From 3ce793dad6135ed115e49505d2339492c6faae05 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 20 Jun 2026 17:04:01 +0000 Subject: [PATCH 01/17] Add usage activity aggregates and activity report scale-up Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/logs_run_processor.go | 24 ++- pkg/cli/logs_usage_activity.go | 94 +++++++++++ pkg/cli/logs_usage_activity_test.go | 69 ++++++++ pkg/workflow/maintenance_workflow_test.go | 4 +- pkg/workflow/maintenance_workflow_yaml.go | 2 +- pkg/workflow/notify_comment.go | 157 ++++++++++++++++++ pkg/workflow/notify_comment_test.go | 6 + pkg/workflow/side_repo_maintenance.go | 2 +- .../side_repo_maintenance_integration_test.go | 4 +- 9 files changed, 348 insertions(+), 14 deletions(-) create mode 100644 pkg/cli/logs_usage_activity.go create mode 100644 pkg/cli/logs_usage_activity_test.go diff --git a/pkg/cli/logs_run_processor.go b/pkg/cli/logs_run_processor.go index f0fe046e24b..381ed662b17 100644 --- a/pkg/cli/logs_run_processor.go +++ b/pkg/cli/logs_run_processor.go @@ -201,6 +201,14 @@ func downloadRunArtifactsConcurrent(ctx context.Context, runs []WorkflowRun, out result.Run.AvgTimeBetweenTurns = metrics.AvgTimeBetweenTurns result.Run.LogsPath = runOutputDir + // Load precomputed activity aggregates from the usage artifact when available. + // These aggregates are generated by the conclusion job and allow lightweight + // usage-only downloads to include firewall/session summaries. + usageActivitySummary, usageActivityErr := loadUsageActivitySummary(runOutputDir) + if usageActivityErr != nil && verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to read usage activity summary for run %d: %v", run.DatabaseID, usageActivityErr))) + } + // If the GitHub API returned an empty workflow path (which can happen for // scheduled or agentic workflow runs), infer it from aw_info.json so that // the cached RunSummary and downstream consumers have a usable identifier. @@ -307,15 +315,13 @@ func downloadRunArtifactsConcurrent(ctx context.Context, runs []WorkflowRun, out result.MCPToolUsage = mcpToolUsage // Analyze token usage from firewall proxy logs. - // Gated on hasFirewallArtifact since token-usage.jsonl lives in the agent artifact. + // token-usage.jsonl is also available in the compact usage artifact. var tokenUsage *TokenUsageSummary - if hasFirewallArtifact { - var tokenErr error - tokenUsage, tokenErr = analyzeTokenUsage(runOutputDir, verbose) - if tokenErr != nil { - if verbose { - fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to analyze token usage for run %d: %v", run.DatabaseID, tokenErr))) - } + var tokenErr error + tokenUsage, tokenErr = analyzeTokenUsage(runOutputDir, verbose) + if tokenErr != nil { + if verbose { + fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to analyze token usage for run %d: %v", run.DatabaseID, tokenErr))) } } result.TokenUsage = tokenUsage @@ -333,6 +339,8 @@ func downloadRunArtifactsConcurrent(ctx context.Context, runs []WorkflowRun, out } } result.GitHubRateLimitUsage = rateLimitUsage + // Fill activity summaries from usage artifact when detailed artifacts were not downloaded. + applyUsageActivitySummaryToResult(usageActivitySummary, &result) // Count safe output items created in GitHub (from manifest artifact) result.Run.SafeItemsCount = len(extractCreatedItemsFromManifest(runOutputDir)) diff --git a/pkg/cli/logs_usage_activity.go b/pkg/cli/logs_usage_activity.go new file mode 100644 index 00000000000..d0c1d914f77 --- /dev/null +++ b/pkg/cli/logs_usage_activity.go @@ -0,0 +1,94 @@ +package cli + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" +) + +type usageActivitySummary struct { + Schema string `json:"schema,omitempty"` + Firewall *usageActivityFirewall `json:"firewall,omitempty"` + Session *usageActivitySession `json:"session,omitempty"` + Gateway *usageActivityGateway `json:"gateway,omitempty"` +} + +type usageActivityFirewall struct { + TotalRequests int `json:"total_requests"` + AllowedRequests int `json:"allowed_requests"` + BlockedRequests int `json:"blocked_requests"` +} + +type usageActivitySession struct { + Turns int `json:"turns"` +} + +type usageActivityGateway struct { + TotalCalls int `json:"total_calls"` + FailedCalls int `json:"failed_calls"` + Servers []usageActivityGatewayServer `json:"servers,omitempty"` +} + +type usageActivityGatewayServer struct { + ServerName string `json:"server_name"` + ToolCallCount int `json:"tool_call_count"` + FailedCalls int `json:"failed_calls"` +} + +func loadUsageActivitySummary(runDir string) (*usageActivitySummary, error) { + candidates := []string{ + filepath.Join(runDir, "usage", "activity", "summary.json"), + filepath.Join(runDir, "activity", "summary.json"), + } + for _, candidate := range candidates { + cleanPath := filepath.Clean(candidate) + raw, err := os.ReadFile(cleanPath) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("read usage activity summary %s: %w", cleanPath, err) + } + var summary usageActivitySummary + if err := json.Unmarshal(raw, &summary); err != nil { + return nil, fmt.Errorf("parse usage activity summary %s: %w", cleanPath, err) + } + return &summary, nil + } + return nil, nil +} + +func applyUsageActivitySummaryToResult(summary *usageActivitySummary, result *DownloadResult) { + if summary == nil || result == nil { + return + } + + if summary.Session != nil && result.Run.Turns == 0 && summary.Session.Turns > 0 { + result.Run.Turns = summary.Session.Turns + } + + if summary.Firewall != nil && result.FirewallAnalysis == nil { + result.FirewallAnalysis = &FirewallAnalysis{ + TotalRequests: summary.Firewall.TotalRequests, + AllowedRequests: summary.Firewall.AllowedRequests, + BlockedRequests: summary.Firewall.BlockedRequests, + RequestsByDomain: map[string]DomainRequestStats{}, + } + } + + if summary.Gateway != nil && result.MCPToolUsage == nil { + servers := make([]MCPServerStats, 0, len(summary.Gateway.Servers)) + for _, server := range summary.Gateway.Servers { + servers = append(servers, MCPServerStats{ + ServerName: server.ServerName, + RequestCount: server.ToolCallCount, + ToolCallCount: server.ToolCallCount, + ErrorCount: server.FailedCalls, + }) + } + result.MCPToolUsage = &MCPToolUsageData{ + Servers: servers, + } + } +} diff --git a/pkg/cli/logs_usage_activity_test.go b/pkg/cli/logs_usage_activity_test.go new file mode 100644 index 00000000000..236ffd7e111 --- /dev/null +++ b/pkg/cli/logs_usage_activity_test.go @@ -0,0 +1,69 @@ +package cli + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadUsageActivitySummary(t *testing.T) { + t.Parallel() + + runDir := t.TempDir() + summaryPath := filepath.Join(runDir, "usage", "activity", "summary.json") + require.NoError(t, os.MkdirAll(filepath.Dir(summaryPath), 0o755)) + require.NoError(t, os.WriteFile(summaryPath, []byte(`{ + "schema":"usage-activity-summary/v1", + "firewall":{"total_requests":10,"allowed_requests":8,"blocked_requests":2}, + "session":{"turns":7}, + "gateway":{"total_calls":5,"failed_calls":1} + }`), 0o644)) + + summary, err := loadUsageActivitySummary(runDir) + require.NoError(t, err) + require.NotNil(t, summary) + require.NotNil(t, summary.Firewall) + assert.Equal(t, 10, summary.Firewall.TotalRequests) + require.NotNil(t, summary.Session) + assert.Equal(t, 7, summary.Session.Turns) + require.NotNil(t, summary.Gateway) + assert.Equal(t, 5, summary.Gateway.TotalCalls) +} + +func TestApplyUsageActivitySummaryToResult(t *testing.T) { + t.Parallel() + + result := DownloadResult{} + summary := &usageActivitySummary{ + Session: &usageActivitySession{Turns: 4}, + Firewall: &usageActivityFirewall{ + TotalRequests: 12, + AllowedRequests: 9, + BlockedRequests: 3, + }, + Gateway: &usageActivityGateway{ + TotalCalls: 6, + FailedCalls: 2, + Servers: []usageActivityGatewayServer{ + {ServerName: "github", ToolCallCount: 5, FailedCalls: 2}, + {ServerName: "playwright", ToolCallCount: 1, FailedCalls: 0}, + }, + }, + } + + applyUsageActivitySummaryToResult(summary, &result) + + assert.Equal(t, 4, result.Run.Turns) + require.NotNil(t, result.FirewallAnalysis) + assert.Equal(t, 12, result.FirewallAnalysis.TotalRequests) + assert.Equal(t, 3, result.FirewallAnalysis.BlockedRequests) + require.NotNil(t, result.MCPToolUsage) + require.Len(t, result.MCPToolUsage.Servers, 2) + assert.Equal(t, "github", result.MCPToolUsage.Servers[0].ServerName) + assert.Equal(t, 5, result.MCPToolUsage.Servers[0].ToolCallCount) + assert.Equal(t, 2, result.MCPToolUsage.Servers[0].ErrorCount) +} + diff --git a/pkg/workflow/maintenance_workflow_test.go b/pkg/workflow/maintenance_workflow_test.go index 934fc8c40df..810522a58db 100644 --- a/pkg/workflow/maintenance_workflow_test.go +++ b/pkg/workflow/maintenance_workflow_test.go @@ -567,8 +567,8 @@ func TestGenerateMaintenanceWorkflow_OperationJobConditions(t *testing.T) { if !strings.Contains(yaml, "--start-date -1w") { t.Errorf("Job activity_report gh aw logs command should include --start-date -1w in:\n%s", yaml) } - if !strings.Contains(yaml, "--count 100") { - t.Errorf("Job activity_report gh aw logs command should include --count 100 in:\n%s", yaml) + if !strings.Contains(yaml, "--count 500") { + t.Errorf("Job activity_report gh aw logs command should include --count 500 in:\n%s", yaml) } if !strings.Contains(yaml, "--format markdown") { t.Errorf("Job activity_report gh aw logs command should include --format markdown in:\n%s", yaml) diff --git a/pkg/workflow/maintenance_workflow_yaml.go b/pkg/workflow/maintenance_workflow_yaml.go index 784ad4f1668..9523fd6c0c2 100644 --- a/pkg/workflow/maintenance_workflow_yaml.go +++ b/pkg/workflow/maintenance_workflow_yaml.go @@ -508,7 +508,7 @@ jobs: ${GH_AW_CMD_PREFIX} logs \ --repo "${{ github.repository }}" \ --start-date -1w \ - --count 100 \ + --count 500 \ --output ./.cache/gh-aw/activity-report-logs \ --format markdown \ --report-file ./.cache/gh-aw/activity-report-logs/report.md diff --git a/pkg/workflow/notify_comment.go b/pkg/workflow/notify_comment.go index 2ca7a7be6b5..50df362228f 100644 --- a/pkg/workflow/notify_comment.go +++ b/pkg/workflow/notify_comment.go @@ -703,6 +703,162 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true\n", " [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl\n", " [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl\n", + " mkdir -p /tmp/gh-aw/usage/activity\n", + " python - <<'PY'\n", + " import glob\n", + " import json\n", + " import os\n", + "\n", + " summary = {'schema': 'usage-activity-summary/v1'}\n", + "\n", + " firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0}\n", + " firewall_paths = [\n", + " '/tmp/gh-aw/sandbox/firewall/logs/*.log',\n", + " '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log',\n", + " '/tmp/gh-aw/squid-logs-*/*.log',\n", + " '/tmp/gh-aw/threat-detection/squid-logs-*/*.log',\n", + " ]\n", + " for pattern in firewall_paths:\n", + " for log_path in glob.glob(pattern):\n", + " try:\n", + " with open(log_path, encoding='utf-8', errors='ignore') as handle:\n", + " for raw in handle:\n", + " line = raw.strip()\n", + " if not line or line.startswith('#'):\n", + " continue\n", + " parts = line.split()\n", + " if len(parts) < 8:\n", + " continue\n", + " firewall['total_requests'] += 1\n", + " status = parts[6]\n", + " decision = parts[7]\n", + " allowed = False\n", + " try:\n", + " code = int(status)\n", + " allowed = code in (200, 206, 304)\n", + " except ValueError:\n", + " allowed = False\n", + " if not allowed and ('TCP_TUNNEL' in decision or 'TCP_HIT' in decision or 'TCP_MISS' in decision):\n", + " allowed = True\n", + " if allowed:\n", + " firewall['allowed_requests'] += 1\n", + " else:\n", + " firewall['blocked_requests'] += 1\n", + " except OSError:\n", + " continue\n", + " if firewall['total_requests'] > 0:\n", + " summary['firewall'] = firewall\n", + "\n", + " session = {\n", + " 'total_events': 0,\n", + " 'session_starts': 0,\n", + " 'session_shutdowns': 0,\n", + " 'turns': 0,\n", + " 'assistant_messages': 0,\n", + " 'reasoning_events': 0,\n", + " 'tool_execution_starts': 0,\n", + " 'tool_execution_completes': 0,\n", + " 'failed_tool_executions': 0,\n", + " }\n", + " session_paths = [\n", + " '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl',\n", + " '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl',\n", + " ]\n", + " for pattern in session_paths:\n", + " for events_path in glob.glob(pattern):\n", + " try:\n", + " with open(events_path, encoding='utf-8', errors='ignore') as handle:\n", + " for raw in handle:\n", + " line = raw.strip()\n", + " if not line or not line.startswith('{'):\n", + " continue\n", + " try:\n", + " entry = json.loads(line)\n", + " except json.JSONDecodeError:\n", + " continue\n", + " event_type = str(entry.get('type', '')).strip().lower()\n", + " session['total_events'] += 1\n", + " if event_type == 'session.start':\n", + " session['session_starts'] += 1\n", + " elif event_type == 'session.shutdown':\n", + " session['session_shutdowns'] += 1\n", + " elif event_type == 'user.message':\n", + " session['turns'] += 1\n", + " elif event_type == 'assistant.message':\n", + " session['assistant_messages'] += 1\n", + " elif event_type in ('reasoning', 'assistant.reasoning'):\n", + " session['reasoning_events'] += 1\n", + " elif event_type == 'tool.execution_start':\n", + " session['tool_execution_starts'] += 1\n", + " elif event_type == 'tool.execution_complete':\n", + " session['tool_execution_completes'] += 1\n", + " data = entry.get('data', {})\n", + " success = True\n", + " if isinstance(data, dict):\n", + " success = bool(data.get('success', True))\n", + " if not success:\n", + " session['failed_tool_executions'] += 1\n", + " except OSError:\n", + " continue\n", + " if session['total_events'] > 0:\n", + " summary['session'] = session\n", + "\n", + " gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}}\n", + " gateway_paths = [\n", + " '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl',\n", + " '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl',\n", + " '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl',\n", + " '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl',\n", + " ]\n", + " for gateway_path in gateway_paths:\n", + " if not os.path.exists(gateway_path):\n", + " continue\n", + " try:\n", + " with open(gateway_path, encoding='utf-8', errors='ignore') as handle:\n", + " for raw in handle:\n", + " line = raw.strip()\n", + " if not line or not line.startswith('{'):\n", + " continue\n", + " try:\n", + " entry = json.loads(line)\n", + " except json.JSONDecodeError:\n", + " continue\n", + " event = str(entry.get('event', '')).strip().lower()\n", + " if event not in ('tool_call', 'rpc_call', 'request'):\n", + " continue\n", + " gateway['total_calls'] += 1\n", + " status = str(entry.get('status', '')).strip().lower()\n", + " level = str(entry.get('level', '')).strip().lower()\n", + " error_text = str(entry.get('error', '')).strip()\n", + " failed = status == 'error' or error_text != '' or level == 'error'\n", + " if failed:\n", + " gateway['failed_calls'] += 1\n", + " server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown')\n", + " server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0})\n", + " server_bucket['tool_call_count'] += 1\n", + " if failed:\n", + " server_bucket['failed_calls'] += 1\n", + " except OSError:\n", + " continue\n", + " if gateway['total_calls'] > 0:\n", + " summary['gateway'] = {\n", + " 'total_calls': gateway['total_calls'],\n", + " 'failed_calls': gateway['failed_calls'],\n", + " 'servers': [\n", + " {\n", + " 'server_name': server_name,\n", + " 'tool_call_count': bucket['tool_call_count'],\n", + " 'failed_calls': bucket['failed_calls'],\n", + " }\n", + " for server_name, bucket in sorted(gateway['servers'].items())\n", + " ],\n", + " }\n", + "\n", + " output_path = '/tmp/gh-aw/usage/activity/summary.json'\n", + " with open(output_path, 'w', encoding='utf-8') as handle:\n", + " json.dump(summary, handle, sort_keys=True)\n", + " print(output_path)\n", + " PY\n", " find /tmp/gh-aw/usage -type f -print | sort\n", " - name: Upload usage artifact\n", " if: always()\n", @@ -716,6 +872,7 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " /tmp/gh-aw/usage/detection_usage.jsonl\n", " /tmp/gh-aw/usage/agent/token_usage.jsonl\n", " /tmp/gh-aw/usage/detection/token_usage.jsonl\n", + " /tmp/gh-aw/usage/activity/summary.json\n", " if-no-files-found: ignore\n", } } diff --git a/pkg/workflow/notify_comment_test.go b/pkg/workflow/notify_comment_test.go index 49baa287984..b62f1e6e9de 100644 --- a/pkg/workflow/notify_comment_test.go +++ b/pkg/workflow/notify_comment_test.go @@ -1213,4 +1213,10 @@ func TestConclusionJobIncludesUsageArtifactSteps(t *testing.T) { if !strings.Contains(allSteps, ": > /tmp/gh-aw/usage/detection/token_usage.jsonl") { t.Errorf("Expected usage artifact collection to ensure detection token usage file exists.\nGenerated steps:\n%s", allSteps) } + if !strings.Contains(allSteps, "python - <<'PY'") { + t.Errorf("Expected usage artifact collection to generate activity summary aggregates.\nGenerated steps:\n%s", allSteps) + } + if !strings.Contains(allSteps, "/tmp/gh-aw/usage/activity/summary.json") { + t.Errorf("Expected usage artifact to include activity summary path.\nGenerated steps:\n%s", allSteps) + } } diff --git a/pkg/workflow/side_repo_maintenance.go b/pkg/workflow/side_repo_maintenance.go index a3c16283cd9..73c75b58ef5 100644 --- a/pkg/workflow/side_repo_maintenance.go +++ b/pkg/workflow/side_repo_maintenance.go @@ -517,7 +517,7 @@ jobs: ${GH_AW_CMD_PREFIX} logs \ --repo "${GH_AW_TARGET_REPO_SLUG}" \ --start-date -1w \ - --count 100 \ + --count 500 \ --output ./.cache/gh-aw/activity-report-logs \ --format markdown \ --report-file ./.cache/gh-aw/activity-report-logs/report.md diff --git a/pkg/workflow/side_repo_maintenance_integration_test.go b/pkg/workflow/side_repo_maintenance_integration_test.go index dc71f9b32a9..cf4dbfcb78c 100644 --- a/pkg/workflow/side_repo_maintenance_integration_test.go +++ b/pkg/workflow/side_repo_maintenance_integration_test.go @@ -116,8 +116,8 @@ This workflow operates on a separate repository. "generated workflow should run gh aw logs directly") assert.Contains(t, contentStr, "--start-date -1w", "generated workflow should download 7 days of logs for activity_report") - assert.Contains(t, contentStr, "--count 100", - "generated workflow should limit activity_report log downloads to at most 100 runs") + assert.Contains(t, contentStr, "--count 500", + "generated workflow should limit activity_report log downloads to at most 500 runs") assert.Contains(t, contentStr, "--format markdown", "generated workflow should request markdown report output from gh aw logs") assert.Contains(t, contentStr, "./.cache/gh-aw/activity-report-logs/report.md", From 6b019aa2af458a0da786a570da75959c589bfe03 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 20 Jun 2026 17:05:16 +0000 Subject: [PATCH 02/17] Format usage activity aggregation files Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/logs_usage_activity.go | 8 ++++---- pkg/cli/logs_usage_activity_test.go | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/cli/logs_usage_activity.go b/pkg/cli/logs_usage_activity.go index d0c1d914f77..f74328c54c3 100644 --- a/pkg/cli/logs_usage_activity.go +++ b/pkg/cli/logs_usage_activity.go @@ -8,10 +8,10 @@ import ( ) type usageActivitySummary struct { - Schema string `json:"schema,omitempty"` - Firewall *usageActivityFirewall `json:"firewall,omitempty"` - Session *usageActivitySession `json:"session,omitempty"` - Gateway *usageActivityGateway `json:"gateway,omitempty"` + Schema string `json:"schema,omitempty"` + Firewall *usageActivityFirewall `json:"firewall,omitempty"` + Session *usageActivitySession `json:"session,omitempty"` + Gateway *usageActivityGateway `json:"gateway,omitempty"` } type usageActivityFirewall struct { diff --git a/pkg/cli/logs_usage_activity_test.go b/pkg/cli/logs_usage_activity_test.go index 236ffd7e111..f5dc3bf0d94 100644 --- a/pkg/cli/logs_usage_activity_test.go +++ b/pkg/cli/logs_usage_activity_test.go @@ -66,4 +66,3 @@ func TestApplyUsageActivitySummaryToResult(t *testing.T) { assert.Equal(t, 5, result.MCPToolUsage.Servers[0].ToolCallCount) assert.Equal(t, 2, result.MCPToolUsage.Servers[0].ErrorCount) } - From 21f6a0eb950d1910465b0eff285f48b21adf0958 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 20 Jun 2026 17:07:06 +0000 Subject: [PATCH 03/17] Address review feedback on usage activity aggregation Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/logs_run_processor.go | 3 ++- pkg/cli/logs_usage_activity.go | 4 +++- pkg/workflow/notify_comment.go | 5 ++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/cli/logs_run_processor.go b/pkg/cli/logs_run_processor.go index 381ed662b17..5bd9d9ddac2 100644 --- a/pkg/cli/logs_run_processor.go +++ b/pkg/cli/logs_run_processor.go @@ -339,7 +339,8 @@ func downloadRunArtifactsConcurrent(ctx context.Context, runs []WorkflowRun, out } } result.GitHubRateLimitUsage = rateLimitUsage - // Fill activity summaries from usage artifact when detailed artifacts were not downloaded. + // Fill missing activity summaries from usage artifact precomputes. + // This call is unconditional but only backfills fields that are still empty. applyUsageActivitySummaryToResult(usageActivitySummary, &result) // Count safe output items created in GitHub (from manifest artifact) diff --git a/pkg/cli/logs_usage_activity.go b/pkg/cli/logs_usage_activity.go index f74328c54c3..f77bb3bf283 100644 --- a/pkg/cli/logs_usage_activity.go +++ b/pkg/cli/logs_usage_activity.go @@ -81,7 +81,9 @@ func applyUsageActivitySummaryToResult(summary *usageActivitySummary, result *Do servers := make([]MCPServerStats, 0, len(summary.Gateway.Servers)) for _, server := range summary.Gateway.Servers { servers = append(servers, MCPServerStats{ - ServerName: server.ServerName, + ServerName: server.ServerName, + // Keep both RequestCount and ToolCallCount aligned because MCPServerStats + // exposes both fields in reports; for usage aggregates we only have call counts. RequestCount: server.ToolCallCount, ToolCallCount: server.ToolCallCount, ErrorCount: server.FailedCalls, diff --git a/pkg/workflow/notify_comment.go b/pkg/workflow/notify_comment.go index 50df362228f..78776759f26 100644 --- a/pkg/workflow/notify_comment.go +++ b/pkg/workflow/notify_comment.go @@ -710,6 +710,7 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " import os\n", "\n", " summary = {'schema': 'usage-activity-summary/v1'}\n", + " allow_decision_markers = ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')\n", "\n", " firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0}\n", " firewall_paths = [\n", @@ -738,7 +739,7 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " allowed = code in (200, 206, 304)\n", " except ValueError:\n", " allowed = False\n", - " if not allowed and ('TCP_TUNNEL' in decision or 'TCP_HIT' in decision or 'TCP_MISS' in decision):\n", + " if not allowed and any(marker in decision for marker in allow_decision_markers):\n", " allowed = True\n", " if allowed:\n", " firewall['allowed_requests'] += 1\n", @@ -786,6 +787,8 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " session['turns'] += 1\n", " elif event_type == 'assistant.message':\n", " session['assistant_messages'] += 1\n", + " # Copilot session logs use both reasoning and assistant.reasoning\n", + " # across CLI/runtime versions, so count both as reasoning events.\n", " elif event_type in ('reasoning', 'assistant.reasoning'):\n", " session['reasoning_events'] += 1\n", " elif event_type == 'tool.execution_start':\n", From 72801be9e38042c3446368921cc9c40e7f07007d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 20 Jun 2026 17:08:53 +0000 Subject: [PATCH 04/17] Incorporate validation feedback comments Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/audit_report.go | 4 +++- pkg/cli/logs_run_processor.go | 4 +--- pkg/workflow/notify_comment.go | 4 ++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/cli/audit_report.go b/pkg/cli/audit_report.go index 475b53280ea..ead40af11e7 100644 --- a/pkg/cli/audit_report.go +++ b/pkg/cli/audit_report.go @@ -198,7 +198,9 @@ type MCPToolCall struct { // MCPServerStats contains server-level statistics type MCPServerStats struct { - ServerName string `json:"server_name" console:"header:Server"` + ServerName string `json:"server_name" console:"header:Server"` + // RequestCount is kept for backward-compatible report schemas that label per-server + // request volume; in MCP usage summaries this currently mirrors ToolCallCount. RequestCount int `json:"request_count" console:"header:Requests"` ToolCallCount int `json:"tool_call_count" console:"header:Tool Calls"` TotalInputSize int `json:"total_input_size" console:"header:Total Input,format:number"` diff --git a/pkg/cli/logs_run_processor.go b/pkg/cli/logs_run_processor.go index 5bd9d9ddac2..c1098a9380a 100644 --- a/pkg/cli/logs_run_processor.go +++ b/pkg/cli/logs_run_processor.go @@ -316,9 +316,7 @@ func downloadRunArtifactsConcurrent(ctx context.Context, runs []WorkflowRun, out // Analyze token usage from firewall proxy logs. // token-usage.jsonl is also available in the compact usage artifact. - var tokenUsage *TokenUsageSummary - var tokenErr error - tokenUsage, tokenErr = analyzeTokenUsage(runOutputDir, verbose) + tokenUsage, tokenErr := analyzeTokenUsage(runOutputDir, verbose) if tokenErr != nil { if verbose { fmt.Fprintln(os.Stderr, console.FormatWarningMessage(fmt.Sprintf("Failed to analyze token usage for run %d: %v", run.DatabaseID, tokenErr))) diff --git a/pkg/workflow/notify_comment.go b/pkg/workflow/notify_comment.go index 78776759f26..974aeda0d53 100644 --- a/pkg/workflow/notify_comment.go +++ b/pkg/workflow/notify_comment.go @@ -709,6 +709,10 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " import json\n", " import os\n", "\n", + " # usage-activity-summary/v1 structure:\n", + " # firewall: total/allowed/blocked request counters\n", + " # session: aggregate Copilot session event counters\n", + " # gateway: total/failed tool-call counters with per-server breakdown\n", " summary = {'schema': 'usage-activity-summary/v1'}\n", " allow_decision_markers = ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')\n", "\n", From e6fd5184bdce97a44c57cc04bd8b8235d2d7f1a5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 20 Jun 2026 17:09:59 +0000 Subject: [PATCH 05/17] Document turn-count backfill precedence Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/logs_usage_activity.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/cli/logs_usage_activity.go b/pkg/cli/logs_usage_activity.go index f77bb3bf283..d8e884d07a0 100644 --- a/pkg/cli/logs_usage_activity.go +++ b/pkg/cli/logs_usage_activity.go @@ -64,6 +64,8 @@ func applyUsageActivitySummaryToResult(summary *usageActivitySummary, result *Do return } + // Preserve previously parsed turn counts (from full session artifacts/events.jsonl) + // and only backfill when they are missing. if summary.Session != nil && result.Run.Turns == 0 && summary.Session.Turns > 0 { result.Run.Turns = summary.Session.Turns } From fc6e46ff84a392c21a9ca27ee8b45e91640ee044 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 20 Jun 2026 17:11:32 +0000 Subject: [PATCH 06/17] Polish comments for aggregate parsing logic Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/logs_usage_activity.go | 1 + pkg/workflow/notify_comment.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/cli/logs_usage_activity.go b/pkg/cli/logs_usage_activity.go index d8e884d07a0..5259fbc366e 100644 --- a/pkg/cli/logs_usage_activity.go +++ b/pkg/cli/logs_usage_activity.go @@ -86,6 +86,7 @@ func applyUsageActivitySummaryToResult(summary *usageActivitySummary, result *Do ServerName: server.ServerName, // Keep both RequestCount and ToolCallCount aligned because MCPServerStats // exposes both fields in reports; for usage aggregates we only have call counts. + // Populate both with the same value to preserve backward-compatible output. RequestCount: server.ToolCallCount, ToolCallCount: server.ToolCallCount, ErrorCount: server.FailedCalls, diff --git a/pkg/workflow/notify_comment.go b/pkg/workflow/notify_comment.go index 974aeda0d53..fa4a209f5ba 100644 --- a/pkg/workflow/notify_comment.go +++ b/pkg/workflow/notify_comment.go @@ -714,7 +714,6 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " # session: aggregate Copilot session event counters\n", " # gateway: total/failed tool-call counters with per-server breakdown\n", " summary = {'schema': 'usage-activity-summary/v1'}\n", - " allow_decision_markers = ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')\n", "\n", " firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0}\n", " firewall_paths = [\n", @@ -735,6 +734,7 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " if len(parts) < 8:\n", " continue\n", " firewall['total_requests'] += 1\n", + " # Squid access log columns: ... method status decision ...\n", " status = parts[6]\n", " decision = parts[7]\n", " allowed = False\n", @@ -743,7 +743,7 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " allowed = code in (200, 206, 304)\n", " except ValueError:\n", " allowed = False\n", - " if not allowed and any(marker in decision for marker in allow_decision_markers):\n", + " if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')):\n", " allowed = True\n", " if allowed:\n", " firewall['allowed_requests'] += 1\n", From 35837917d359a25db18889f672289f127ddf39d1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 20 Jun 2026 17:13:09 +0000 Subject: [PATCH 07/17] Align usage session schema and parsing comments Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/logs_usage_activity.go | 10 +++++++++- pkg/workflow/notify_comment.go | 9 +++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/pkg/cli/logs_usage_activity.go b/pkg/cli/logs_usage_activity.go index 5259fbc366e..8790e1e2028 100644 --- a/pkg/cli/logs_usage_activity.go +++ b/pkg/cli/logs_usage_activity.go @@ -21,7 +21,15 @@ type usageActivityFirewall struct { } type usageActivitySession struct { - Turns int `json:"turns"` + TotalEvents int `json:"total_events"` + SessionStarts int `json:"session_starts"` + SessionShutdowns int `json:"session_shutdowns"` + Turns int `json:"turns"` + AssistantMessages int `json:"assistant_messages"` + ReasoningEvents int `json:"reasoning_events"` + ToolExecutionStarts int `json:"tool_execution_starts"` + ToolExecutionCompletes int `json:"tool_execution_completes"` + FailedToolExecutions int `json:"failed_tool_executions"` } type usageActivityGateway struct { diff --git a/pkg/workflow/notify_comment.go b/pkg/workflow/notify_comment.go index fa4a209f5ba..40dc2c09805 100644 --- a/pkg/workflow/notify_comment.go +++ b/pkg/workflow/notify_comment.go @@ -714,6 +714,8 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " # session: aggregate Copilot session event counters\n", " # gateway: total/failed tool-call counters with per-server breakdown\n", " summary = {'schema': 'usage-activity-summary/v1'}\n", + " SQUID_STATUS_INDEX = 6\n", + " SQUID_DECISION_INDEX = 7\n", "\n", " firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0}\n", " firewall_paths = [\n", @@ -735,8 +737,9 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " continue\n", " firewall['total_requests'] += 1\n", " # Squid access log columns: ... method status decision ...\n", - " status = parts[6]\n", - " decision = parts[7]\n", + " # Keep indices named for easier maintenance if format changes.\n", + " status = parts[SQUID_STATUS_INDEX]\n", + " decision = parts[SQUID_DECISION_INDEX]\n", " allowed = False\n", " try:\n", " code = int(status)\n", @@ -840,6 +843,8 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " failed = status == 'error' or error_text != '' or level == 'error'\n", " if failed:\n", " gateway['failed_calls'] += 1\n", + " # gateway.jsonl has server_name for modern logs and server_id in\n", + " # some compatibility/transition paths; keep fallback ordering explicit.\n", " server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown')\n", " server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0})\n", " server_bucket['tool_call_count'] += 1\n", From 28c3468f46e721d6e28085c3052043830ae96330 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 20 Jun 2026 17:14:27 +0000 Subject: [PATCH 08/17] Clarify inline aggregate script and squid columns Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/workflow/notify_comment.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/workflow/notify_comment.go b/pkg/workflow/notify_comment.go index 40dc2c09805..5e893dc09d1 100644 --- a/pkg/workflow/notify_comment.go +++ b/pkg/workflow/notify_comment.go @@ -709,6 +709,9 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " import json\n", " import os\n", "\n", + " # NOTE: this aggregation script intentionally stays inline in the generated\n", + " # workflow step so compiled workflows are self-contained and do not depend on\n", + " # extra repository files at runtime.\n", " # usage-activity-summary/v1 structure:\n", " # firewall: total/allowed/blocked request counters\n", " # session: aggregate Copilot session event counters\n", @@ -736,7 +739,9 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " if len(parts) < 8:\n", " continue\n", " firewall['total_requests'] += 1\n", - " # Squid access log columns: ... method status decision ...\n", + " # Squid access log columns (0-based):\n", + " # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method\n", + " # 6=status 7=decision 8=url 9=user-agent\n", " # Keep indices named for easier maintenance if format changes.\n", " status = parts[SQUID_STATUS_INDEX]\n", " decision = parts[SQUID_DECISION_INDEX]\n", From da6f6ed857bcd672c5bf2b7f667ce3d68fcc3a51 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 20 Jun 2026 17:16:05 +0000 Subject: [PATCH 09/17] Switch aggregate step to python3 and clarify MCP counts Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/logs_usage_activity.go | 5 +++-- pkg/workflow/notify_comment.go | 2 +- pkg/workflow/notify_comment_test.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/cli/logs_usage_activity.go b/pkg/cli/logs_usage_activity.go index 8790e1e2028..0604bb29c45 100644 --- a/pkg/cli/logs_usage_activity.go +++ b/pkg/cli/logs_usage_activity.go @@ -93,8 +93,9 @@ func applyUsageActivitySummaryToResult(summary *usageActivitySummary, result *Do servers = append(servers, MCPServerStats{ ServerName: server.ServerName, // Keep both RequestCount and ToolCallCount aligned because MCPServerStats - // exposes both fields in reports; for usage aggregates we only have call counts. - // Populate both with the same value to preserve backward-compatible output. + // distinguishes overall request volume (RequestCount) from tool-invocation + // volume (ToolCallCount). In usage-aggregate mode we only have per-server + // tool-call counts, so both fields are populated from that single source. RequestCount: server.ToolCallCount, ToolCallCount: server.ToolCallCount, ErrorCount: server.FailedCalls, diff --git a/pkg/workflow/notify_comment.go b/pkg/workflow/notify_comment.go index 5e893dc09d1..ac6e8b420bd 100644 --- a/pkg/workflow/notify_comment.go +++ b/pkg/workflow/notify_comment.go @@ -704,7 +704,7 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl\n", " [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl\n", " mkdir -p /tmp/gh-aw/usage/activity\n", - " python - <<'PY'\n", + " python3 - <<'PY'\n", " import glob\n", " import json\n", " import os\n", diff --git a/pkg/workflow/notify_comment_test.go b/pkg/workflow/notify_comment_test.go index b62f1e6e9de..0a0cc9149db 100644 --- a/pkg/workflow/notify_comment_test.go +++ b/pkg/workflow/notify_comment_test.go @@ -1213,7 +1213,7 @@ func TestConclusionJobIncludesUsageArtifactSteps(t *testing.T) { if !strings.Contains(allSteps, ": > /tmp/gh-aw/usage/detection/token_usage.jsonl") { t.Errorf("Expected usage artifact collection to ensure detection token usage file exists.\nGenerated steps:\n%s", allSteps) } - if !strings.Contains(allSteps, "python - <<'PY'") { + if !strings.Contains(allSteps, "python3 - <<'PY'") { t.Errorf("Expected usage artifact collection to generate activity summary aggregates.\nGenerated steps:\n%s", allSteps) } if !strings.Contains(allSteps, "/tmp/gh-aw/usage/activity/summary.json") { From 6d696aeddcc3fb22beacf4705ebdac286a32a254 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 20 Jun 2026 19:59:27 +0000 Subject: [PATCH 10/17] Add draft ADR-40504 for usage-artifact activity aggregates Co-Authored-By: Claude Opus 4.8 (1M context) --- ...pute-usage-artifact-activity-aggregates.md | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 docs/adr/40504-precompute-usage-artifact-activity-aggregates.md diff --git a/docs/adr/40504-precompute-usage-artifact-activity-aggregates.md b/docs/adr/40504-precompute-usage-artifact-activity-aggregates.md new file mode 100644 index 00000000000..14971423c66 --- /dev/null +++ b/docs/adr/40504-precompute-usage-artifact-activity-aggregates.md @@ -0,0 +1,40 @@ +# ADR-40504: Precompute Usage-Artifact Activity Aggregates for Lightweight Logs Reporting + +**Date**: 2026-06-20 +**Status**: Draft + +## Context + +`gh aw logs` activity reporting derives firewall, session, and MCP-gateway statistics by downloading and parsing large per-run agent artifacts. This is expensive and does not scale to wide run windows, which previously capped maintenance activity reports at `--count 100`. The conclusion job already uploads a compact `usage` artifact, making it a natural place to precompute lighter-weight summaries. The goal is richer activity reporting over a larger run window (raised to `--count 500`) without paying the cost of full agent-artifact downloads. + +## Decision + +We will have the conclusion job precompute run-activity aggregates into the `usage` artifact at `usage/activity/summary.json`, tagged with the schema marker `usage-activity-summary/v1`. The aggregation runs as an inline `python3` step embedded in the generated workflow (so compiled workflows stay self-contained with no extra repository-file dependency) and rolls up firewall request counts, Copilot session event counters, and gateway tool-call counters with a per-server breakdown. The logs pipeline (`pkg/cli/logs_usage_activity.go`) loads this summary and backfills only fields that are still empty after normal artifact parsing, keeping precedence explicit: detailed artifacts win, the usage summary fills gaps only. + +## Alternatives Considered + +### Alternative 1: Keep computing activity stats client-side from full agent artifacts +This is the status quo. It was rejected because downloading and parsing full agent artifacts for every run is too heavy to scale to a 500-run activity window; the lighter usage-only path is what enables the increased window. + +### Alternative 2: Ship the aggregation logic as a committed repository script instead of inline Python +A separate script file would be easier to test and review. It was rejected to keep compiled/generated workflows self-contained — runtime steps must not depend on additional repository files that may not be present in the execution environment. + +## Consequences + +### Positive +- Activity reports can process a much larger run window (`--count` raised from 100 to 500) using only the compact usage artifact. +- Compiled workflows remain self-contained because the aggregation script is embedded inline. +- Backfill precedence is explicit, so detailed-artifact data is never overwritten by the coarser summary. + +### Negative +- The inline Python aggregation script is duplicated into every generated workflow and cannot be unit-tested as a standalone module. +- Fidelity is reduced in usage-aggregate mode: per-server `RequestCount` is populated from `ToolCallCount` because only tool-call counts are available, blurring the request-vs-tool-call distinction. +- A new artifact schema (`usage-activity-summary/v1`) must be versioned and kept in sync between the producer (`notify_comment.go`) and consumer (`logs_usage_activity.go`). + +### Neutral +- Firewall allowed/blocked classification depends on Squid access-log column positions and status-code heuristics, which are tied to the current log format. +- The summary is only emitted when underlying logs exist; runs without firewall/session/gateway logs simply omit those sections. + +--- + +*This is a DRAFT ADR generated by the [Design Decision Gate](https://github.com/github/gh-aw/actions/runs/27882101861) workflow. The PR author must review, complete, and finalize this document before the PR can merge.* From e4310ec9cf2a3c8c75e71bdd41dfb4cc475842ac Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 20 Jun 2026 20:44:44 +0000 Subject: [PATCH 11/17] Plan PR finisher pass Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/ab-testing-advisor.lock.yml | 174 ++++++++++++++++++ .github/workflows/ace-editor.lock.yml | 174 ++++++++++++++++++ .../agent-performance-analyzer.lock.yml | 174 ++++++++++++++++++ .../workflows/agent-persona-explorer.lock.yml | 174 ++++++++++++++++++ .../workflows/agentic-token-audit.lock.yml | 174 ++++++++++++++++++ .../agentic-token-optimizer.lock.yml | 174 ++++++++++++++++++ .../agentic-token-trend-audit.lock.yml | 174 ++++++++++++++++++ .github/workflows/agentics-maintenance.yml | 2 +- .github/workflows/ai-moderator.lock.yml | 174 ++++++++++++++++++ .../workflows/api-consumption-report.lock.yml | 174 ++++++++++++++++++ .github/workflows/approach-validator.lock.yml | 174 ++++++++++++++++++ .github/workflows/archie.lock.yml | 174 ++++++++++++++++++ .../workflows/architecture-guardian.lock.yml | 174 ++++++++++++++++++ .github/workflows/artifacts-summary.lock.yml | 174 ++++++++++++++++++ .github/workflows/audit-workflows.lock.yml | 174 ++++++++++++++++++ .github/workflows/auto-triage-issues.lock.yml | 174 ++++++++++++++++++ .github/workflows/avenger.lock.yml | 174 ++++++++++++++++++ .../aw-failure-investigator.lock.yml | 174 ++++++++++++++++++ .github/workflows/blog-auditor.lock.yml | 174 ++++++++++++++++++ .github/workflows/bot-detection.lock.yml | 174 ++++++++++++++++++ .github/workflows/brave.lock.yml | 174 ++++++++++++++++++ .../breaking-change-checker.lock.yml | 174 ++++++++++++++++++ .github/workflows/changeset.lock.yml | 174 ++++++++++++++++++ .../workflows/chaos-pr-bundle-fuzzer.lock.yml | 174 ++++++++++++++++++ .github/workflows/ci-coach.lock.yml | 174 ++++++++++++++++++ .github/workflows/ci-doctor.lock.yml | 174 ++++++++++++++++++ .../claude-code-user-docs-review.lock.yml | 174 ++++++++++++++++++ .../cli-consistency-checker.lock.yml | 174 ++++++++++++++++++ .../workflows/cli-version-checker.lock.yml | 174 ++++++++++++++++++ .github/workflows/cloclo.lock.yml | 174 ++++++++++++++++++ .../workflows/code-scanning-fixer.lock.yml | 174 ++++++++++++++++++ .github/workflows/code-simplifier.lock.yml | 174 ++++++++++++++++++ .../codex-github-remote-mcp-test.lock.yml | 174 ++++++++++++++++++ .../commit-changes-analyzer.lock.yml | 174 ++++++++++++++++++ .../constraint-solving-potd.lock.yml | 174 ++++++++++++++++++ .github/workflows/contribution-check.lock.yml | 174 ++++++++++++++++++ .../workflows/copilot-agent-analysis.lock.yml | 174 ++++++++++++++++++ .../copilot-centralization-drilldown.lock.yml | 174 ++++++++++++++++++ .../copilot-centralization-optimizer.lock.yml | 174 ++++++++++++++++++ .../copilot-cli-deep-research.lock.yml | 174 ++++++++++++++++++ .github/workflows/copilot-opt.lock.yml | 174 ++++++++++++++++++ .../copilot-pr-merged-report.lock.yml | 174 ++++++++++++++++++ .../copilot-pr-nlp-analysis.lock.yml | 174 ++++++++++++++++++ .../copilot-pr-prompt-analysis.lock.yml | 174 ++++++++++++++++++ .../copilot-session-insights.lock.yml | 174 ++++++++++++++++++ .github/workflows/craft.lock.yml | 174 ++++++++++++++++++ ...aily-agent-of-the-day-blog-writer.lock.yml | 174 ++++++++++++++++++ .../daily-agentrx-trace-optimizer.lock.yml | 174 ++++++++++++++++++ .../daily-ambient-context-optimizer.lock.yml | 174 ++++++++++++++++++ .../daily-architecture-diagram.lock.yml | 174 ++++++++++++++++++ .../daily-assign-issue-to-user.lock.yml | 174 ++++++++++++++++++ ...strostylelite-markdown-spellcheck.lock.yml | 174 ++++++++++++++++++ ...daily-aw-cross-repo-compile-check.lock.yml | 174 ++++++++++++++++++ ...daily-awf-spec-compiler-surfacing.lock.yml | 174 ++++++++++++++++++ .../workflows/daily-byok-ollama-test.lock.yml | 174 ++++++++++++++++++ .../daily-cache-strategy-analyzer.lock.yml | 174 ++++++++++++++++++ .../daily-caveman-optimizer.lock.yml | 174 ++++++++++++++++++ .github/workflows/daily-choice-test.lock.yml | 174 ++++++++++++++++++ .../workflows/daily-cli-performance.lock.yml | 174 ++++++++++++++++++ .../workflows/daily-cli-tools-tester.lock.yml | 174 ++++++++++++++++++ .github/workflows/daily-code-metrics.lock.yml | 174 ++++++++++++++++++ .../daily-community-attribution.lock.yml | 174 ++++++++++++++++++ .../workflows/daily-compiler-quality.lock.yml | 174 ++++++++++++++++++ ...ly-compiler-threat-spec-optimizer.lock.yml | 174 ++++++++++++++++++ .../daily-credit-limit-test.lock.yml | 174 ++++++++++++++++++ .github/workflows/daily-doc-healer.lock.yml | 174 ++++++++++++++++++ .github/workflows/daily-doc-updater.lock.yml | 174 ++++++++++++++++++ .../daily-experiment-report.lock.yml | 174 ++++++++++++++++++ .github/workflows/daily-fact.lock.yml | 174 ++++++++++++++++++ .github/workflows/daily-file-diet.lock.yml | 174 ++++++++++++++++++ .../workflows/daily-firewall-report.lock.yml | 174 ++++++++++++++++++ .../daily-formal-spec-verifier.lock.yml | 174 ++++++++++++++++++ .../workflows/daily-function-namer.lock.yml | 174 ++++++++++++++++++ .../workflows/daily-geo-optimizer.lock.yml | 174 ++++++++++++++++++ .github/workflows/daily-hippo-learn.lock.yml | 174 ++++++++++++++++++ .../workflows/daily-issues-report.lock.yml | 174 ++++++++++++++++++ .../daily-malicious-code-scan.lock.yml | 174 ++++++++++++++++++ .../daily-max-ai-credits-test.lock.yml | 174 ++++++++++++++++++ .../daily-mcp-concurrency-analysis.lock.yml | 174 ++++++++++++++++++ .../workflows/daily-model-inventory.lock.yml | 174 ++++++++++++++++++ .../daily-multi-device-docs-tester.lock.yml | 174 ++++++++++++++++++ .github/workflows/daily-news.lock.yml | 174 ++++++++++++++++++ .../daily-observability-report.lock.yml | 174 ++++++++++++++++++ .../daily-performance-summary.lock.yml | 174 ++++++++++++++++++ .github/workflows/daily-regulatory.lock.yml | 174 ++++++++++++++++++ .../daily-reliability-review.lock.yml | 174 ++++++++++++++++++ .../daily-rendering-scripts-verifier.lock.yml | 174 ++++++++++++++++++ .../workflows/daily-repo-chronicle.lock.yml | 174 ++++++++++++++++++ .../daily-safe-output-integrator.lock.yml | 174 ++++++++++++++++++ .../daily-safe-output-optimizer.lock.yml | 174 ++++++++++++++++++ .../daily-safe-outputs-conformance.lock.yml | 174 ++++++++++++++++++ .../daily-safeoutputs-git-simulator.lock.yml | 174 ++++++++++++++++++ .../workflows/daily-secrets-analysis.lock.yml | 174 ++++++++++++++++++ .../daily-security-observability.lock.yml | 174 ++++++++++++++++++ .../daily-security-red-team.lock.yml | 174 ++++++++++++++++++ .github/workflows/daily-semgrep-scan.lock.yml | 174 ++++++++++++++++++ .../workflows/daily-sentrux-report.lock.yml | 174 ++++++++++++++++++ .../workflows/daily-skill-optimizer.lock.yml | 174 ++++++++++++++++++ .../daily-spdd-spec-planner.lock.yml | 174 ++++++++++++++++++ .../daily-syntax-error-quality.lock.yml | 174 ++++++++++++++++++ .../daily-team-evolution-insights.lock.yml | 174 ++++++++++++++++++ .github/workflows/daily-team-status.lock.yml | 174 ++++++++++++++++++ .../daily-testify-uber-super-expert.lock.yml | 174 ++++++++++++++++++ .../daily-token-consumption-report.lock.yml | 174 ++++++++++++++++++ ...dows-terminal-integration-builder.lock.yml | 174 ++++++++++++++++++ .../workflows/daily-workflow-updater.lock.yml | 174 ++++++++++++++++++ .../dataflow-pr-discussion-dataset.lock.yml | 174 ++++++++++++++++++ .github/workflows/dead-code-remover.lock.yml | 174 ++++++++++++++++++ .github/workflows/deep-report.lock.yml | 174 ++++++++++++++++++ .github/workflows/delight.lock.yml | 174 ++++++++++++++++++ .github/workflows/dependabot-burner.lock.yml | 174 ++++++++++++++++++ .../workflows/dependabot-go-checker.lock.yml | 174 ++++++++++++++++++ .github/workflows/dependabot-repair.lock.yml | 174 ++++++++++++++++++ .../deployment-incident-monitor.lock.yml | 174 ++++++++++++++++++ .../workflows/design-decision-gate.lock.yml | 174 ++++++++++++++++++ .../workflows/designer-drift-audit.lock.yml | 174 ++++++++++++++++++ .github/workflows/dev-hawk.lock.yml | 174 ++++++++++++++++++ .github/workflows/dev.lock.yml | 174 ++++++++++++++++++ .../developer-docs-consolidator.lock.yml | 174 ++++++++++++++++++ .github/workflows/dictation-prompt.lock.yml | 174 ++++++++++++++++++ .../workflows/discussion-task-miner.lock.yml | 174 ++++++++++++++++++ .github/workflows/docs-noob-tester.lock.yml | 174 ++++++++++++++++++ .github/workflows/draft-pr-cleanup.lock.yml | 174 ++++++++++++++++++ .../duplicate-code-detector.lock.yml | 174 ++++++++++++++++++ .../example-failure-category-filter.lock.yml | 174 ++++++++++++++++++ .../example-permissions-warning.lock.yml | 174 ++++++++++++++++++ .../example-workflow-analyzer.lock.yml | 174 ++++++++++++++++++ .github/workflows/firewall-escape.lock.yml | 174 ++++++++++++++++++ .github/workflows/firewall.lock.yml | 174 ++++++++++++++++++ .../workflows/functional-pragmatist.lock.yml | 174 ++++++++++++++++++ .../github-mcp-structural-analysis.lock.yml | 174 ++++++++++++++++++ .../github-mcp-tools-report.lock.yml | 174 ++++++++++++++++++ .../github-remote-mcp-auth-test.lock.yml | 174 ++++++++++++++++++ .../workflows/glossary-maintainer.lock.yml | 174 ++++++++++++++++++ .github/workflows/go-fan.lock.yml | 174 ++++++++++++++++++ .github/workflows/go-logger.lock.yml | 174 ++++++++++++++++++ .../workflows/go-pattern-detector.lock.yml | 174 ++++++++++++++++++ .github/workflows/gpclean.lock.yml | 174 ++++++++++++++++++ .github/workflows/grumpy-reviewer.lock.yml | 174 ++++++++++++++++++ .github/workflows/hippo-embed.lock.yml | 174 ++++++++++++++++++ .github/workflows/hourly-ci-cleaner.lock.yml | 174 ++++++++++++++++++ .../workflows/instructions-janitor.lock.yml | 174 ++++++++++++++++++ .github/workflows/issue-arborist.lock.yml | 174 ++++++++++++++++++ .github/workflows/issue-monster.lock.yml | 174 ++++++++++++++++++ .github/workflows/issue-triage-agent.lock.yml | 174 ++++++++++++++++++ .github/workflows/jsweep.lock.yml | 174 ++++++++++++++++++ .../workflows/layout-spec-maintainer.lock.yml | 174 ++++++++++++++++++ .github/workflows/lint-monster.lock.yml | 174 ++++++++++++++++++ .github/workflows/linter-miner.lock.yml | 174 ++++++++++++++++++ .github/workflows/lockfile-stats.lock.yml | 174 ++++++++++++++++++ .../mattpocock-skills-reviewer.lock.yml | 174 ++++++++++++++++++ .github/workflows/mcp-inspector.lock.yml | 174 ++++++++++++++++++ .github/workflows/mergefest.lock.yml | 174 ++++++++++++++++++ .github/workflows/metrics-collector.lock.yml | 174 ++++++++++++++++++ .github/workflows/necromancer.lock.yml | 174 ++++++++++++++++++ .../workflows/notion-issue-summary.lock.yml | 174 ++++++++++++++++++ .../objective-impact-report.lock.yml | 174 ++++++++++++++++++ .github/workflows/org-health-report.lock.yml | 174 ++++++++++++++++++ .github/workflows/outcome-collector.lock.yml | 174 ++++++++++++++++++ .github/workflows/pdf-summary.lock.yml | 174 ++++++++++++++++++ .github/workflows/plan.lock.yml | 174 ++++++++++++++++++ .github/workflows/poem-bot.lock.yml | 174 ++++++++++++++++++ .github/workflows/portfolio-analyst.lock.yml | 174 ++++++++++++++++++ .../pr-code-quality-reviewer.lock.yml | 174 ++++++++++++++++++ .../workflows/pr-description-caveman.lock.yml | 174 ++++++++++++++++++ .../workflows/pr-nitpick-reviewer.lock.yml | 174 ++++++++++++++++++ .github/workflows/pr-sous-chef.lock.yml | 174 ++++++++++++++++++ .github/workflows/pr-triage-agent.lock.yml | 174 ++++++++++++++++++ .../prompt-clustering-analysis.lock.yml | 174 ++++++++++++++++++ .github/workflows/python-data-charts.lock.yml | 174 ++++++++++++++++++ .github/workflows/q.lock.yml | 174 ++++++++++++++++++ .../workflows/refactoring-cadence.lock.yml | 174 ++++++++++++++++++ .github/workflows/refiner.lock.yml | 174 ++++++++++++++++++ .github/workflows/release.lock.yml | 174 ++++++++++++++++++ .../workflows/repo-audit-analyzer.lock.yml | 174 ++++++++++++++++++ .github/workflows/repo-tree-map.lock.yml | 174 ++++++++++++++++++ .../repository-quality-improver.lock.yml | 174 ++++++++++++++++++ .github/workflows/research.lock.yml | 174 ++++++++++++++++++ .github/workflows/ruflo-backed-task.lock.yml | 174 ++++++++++++++++++ .github/workflows/safe-output-health.lock.yml | 174 ++++++++++++++++++ .../schema-consistency-checker.lock.yml | 174 ++++++++++++++++++ .../schema-feature-coverage.lock.yml | 174 ++++++++++++++++++ .github/workflows/scout.lock.yml | 174 ++++++++++++++++++ .../workflows/security-compliance.lock.yml | 174 ++++++++++++++++++ .github/workflows/security-review.lock.yml | 174 ++++++++++++++++++ .../semantic-function-refactor.lock.yml | 174 ++++++++++++++++++ .github/workflows/sergo.lock.yml | 174 ++++++++++++++++++ .github/workflows/skillet.lock.yml | 174 ++++++++++++++++++ .../workflows/slide-deck-maintainer.lock.yml | 174 ++++++++++++++++++ .../workflows/smoke-agent-all-merged.lock.yml | 174 ++++++++++++++++++ .../workflows/smoke-agent-all-none.lock.yml | 174 ++++++++++++++++++ .../smoke-agent-public-approved.lock.yml | 174 ++++++++++++++++++ .../smoke-agent-public-none.lock.yml | 174 ++++++++++++++++++ .../smoke-agent-scoped-approved.lock.yml | 174 ++++++++++++++++++ .github/workflows/smoke-antigravity.lock.yml | 174 ++++++++++++++++++ .../workflows/smoke-call-workflow.lock.yml | 174 ++++++++++++++++++ .github/workflows/smoke-ci.lock.yml | 174 ++++++++++++++++++ .github/workflows/smoke-claude.lock.yml | 174 ++++++++++++++++++ .github/workflows/smoke-codex.lock.yml | 174 ++++++++++++++++++ .../smoke-copilot-aoai-apikey.lock.yml | 174 ++++++++++++++++++ .../smoke-copilot-aoai-entra.lock.yml | 174 ++++++++++++++++++ .github/workflows/smoke-copilot-arm.lock.yml | 174 ++++++++++++++++++ .github/workflows/smoke-copilot-sdk.lock.yml | 174 ++++++++++++++++++ .github/workflows/smoke-copilot.lock.yml | 174 ++++++++++++++++++ .../smoke-create-cross-repo-pr.lock.yml | 174 ++++++++++++++++++ .github/workflows/smoke-crush.lock.yml | 174 ++++++++++++++++++ .github/workflows/smoke-gemini.lock.yml | 174 ++++++++++++++++++ .github/workflows/smoke-multi-pr.lock.yml | 174 ++++++++++++++++++ .github/workflows/smoke-opencode.lock.yml | 174 ++++++++++++++++++ .../workflows/smoke-otel-backends.lock.yml | 174 ++++++++++++++++++ .github/workflows/smoke-pi.lock.yml | 174 ++++++++++++++++++ .github/workflows/smoke-project.lock.yml | 174 ++++++++++++++++++ .../workflows/smoke-service-ports.lock.yml | 174 ++++++++++++++++++ .github/workflows/smoke-temporary-id.lock.yml | 174 ++++++++++++++++++ .github/workflows/smoke-test-tools.lock.yml | 174 ++++++++++++++++++ .../smoke-update-cross-repo-pr.lock.yml | 174 ++++++++++++++++++ .../smoke-workflow-call-with-inputs.lock.yml | 174 ++++++++++++++++++ .../workflows/smoke-workflow-call.lock.yml | 174 ++++++++++++++++++ .github/workflows/spec-enforcer.lock.yml | 174 ++++++++++++++++++ .github/workflows/spec-extractor.lock.yml | 174 ++++++++++++++++++ .github/workflows/spec-librarian.lock.yml | 174 ++++++++++++++++++ .github/workflows/stale-pr-cleanup.lock.yml | 174 ++++++++++++++++++ .../workflows/stale-repo-identifier.lock.yml | 174 ++++++++++++++++++ .../workflows/static-analysis-report.lock.yml | 174 ++++++++++++++++++ .../workflows/step-name-alignment.lock.yml | 174 ++++++++++++++++++ .github/workflows/sub-issue-closer.lock.yml | 174 ++++++++++++++++++ .github/workflows/super-linter.lock.yml | 174 ++++++++++++++++++ .../workflows/technical-doc-writer.lock.yml | 174 ++++++++++++++++++ .github/workflows/terminal-stylist.lock.yml | 174 ++++++++++++++++++ .../test-create-pr-error-handling.lock.yml | 174 ++++++++++++++++++ .github/workflows/test-dispatcher.lock.yml | 174 ++++++++++++++++++ .../test-project-url-default.lock.yml | 174 ++++++++++++++++++ .../workflows/test-quality-sentinel.lock.yml | 174 ++++++++++++++++++ .github/workflows/test-workflow.lock.yml | 174 ++++++++++++++++++ .github/workflows/tidy.lock.yml | 174 ++++++++++++++++++ .github/workflows/typist.lock.yml | 174 ++++++++++++++++++ .../workflows/ubuntu-image-analyzer.lock.yml | 174 ++++++++++++++++++ .../uk-ai-operational-resilience.lock.yml | 174 ++++++++++++++++++ .github/workflows/unbloat-docs.lock.yml | 174 ++++++++++++++++++ .github/workflows/update-astro.lock.yml | 174 ++++++++++++++++++ .github/workflows/video-analyzer.lock.yml | 174 ++++++++++++++++++ .../visual-regression-checker.lock.yml | 174 ++++++++++++++++++ .../weekly-blog-post-writer.lock.yml | 174 ++++++++++++++++++ .../weekly-editors-health-check.lock.yml | 174 ++++++++++++++++++ .../workflows/weekly-issue-summary.lock.yml | 174 ++++++++++++++++++ .../weekly-safe-outputs-spec-review.lock.yml | 174 ++++++++++++++++++ .github/workflows/workflow-generator.lock.yml | 174 ++++++++++++++++++ .../workflow-health-manager.lock.yml | 174 ++++++++++++++++++ .../workflows/workflow-normalizer.lock.yml | 174 ++++++++++++++++++ .../workflow-skill-extractor.lock.yml | 174 ++++++++++++++++++ 250 files changed, 43327 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ab-testing-advisor.lock.yml b/.github/workflows/ab-testing-advisor.lock.yml index 6e6d05573ca..1ef169c6591 100644 --- a/.github/workflows/ab-testing-advisor.lock.yml +++ b/.github/workflows/ab-testing-advisor.lock.yml @@ -1112,6 +1112,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1125,6 +1298,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/ace-editor.lock.yml b/.github/workflows/ace-editor.lock.yml index acccbd531cd..609b12efa48 100644 --- a/.github/workflows/ace-editor.lock.yml +++ b/.github/workflows/ace-editor.lock.yml @@ -1050,6 +1050,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1063,6 +1236,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/agent-performance-analyzer.lock.yml b/.github/workflows/agent-performance-analyzer.lock.yml index dad06ac06cf..759282e189a 100644 --- a/.github/workflows/agent-performance-analyzer.lock.yml +++ b/.github/workflows/agent-performance-analyzer.lock.yml @@ -1319,6 +1319,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1332,6 +1505,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/agent-persona-explorer.lock.yml b/.github/workflows/agent-persona-explorer.lock.yml index e31a3394c8d..f94056d6b2d 100644 --- a/.github/workflows/agent-persona-explorer.lock.yml +++ b/.github/workflows/agent-persona-explorer.lock.yml @@ -1236,6 +1236,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1249,6 +1422,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/agentic-token-audit.lock.yml b/.github/workflows/agentic-token-audit.lock.yml index 2b88f7126b5..1fe4ea0875e 100644 --- a/.github/workflows/agentic-token-audit.lock.yml +++ b/.github/workflows/agentic-token-audit.lock.yml @@ -1250,6 +1250,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1263,6 +1436,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/agentic-token-optimizer.lock.yml b/.github/workflows/agentic-token-optimizer.lock.yml index 3abcedeb2ec..0f76d6c1e3d 100644 --- a/.github/workflows/agentic-token-optimizer.lock.yml +++ b/.github/workflows/agentic-token-optimizer.lock.yml @@ -1117,6 +1117,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1130,6 +1303,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/agentic-token-trend-audit.lock.yml b/.github/workflows/agentic-token-trend-audit.lock.yml index cc6b43bfaca..b8014258f95 100644 --- a/.github/workflows/agentic-token-trend-audit.lock.yml +++ b/.github/workflows/agentic-token-trend-audit.lock.yml @@ -1207,6 +1207,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1220,6 +1393,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/agentics-maintenance.yml b/.github/workflows/agentics-maintenance.yml index 75e7134023e..a82eeffc6e0 100644 --- a/.github/workflows/agentics-maintenance.yml +++ b/.github/workflows/agentics-maintenance.yml @@ -424,7 +424,7 @@ jobs: ${GH_AW_CMD_PREFIX} logs \ --repo "${{ github.repository }}" \ --start-date -1w \ - --count 100 \ + --count 500 \ --output ./.cache/gh-aw/activity-report-logs \ --format markdown \ --report-file ./.cache/gh-aw/activity-report-logs/report.md diff --git a/.github/workflows/ai-moderator.lock.yml b/.github/workflows/ai-moderator.lock.yml index 3e67f24d6cc..f25d22461ee 100644 --- a/.github/workflows/ai-moderator.lock.yml +++ b/.github/workflows/ai-moderator.lock.yml @@ -1240,6 +1240,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1253,6 +1426,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/api-consumption-report.lock.yml b/.github/workflows/api-consumption-report.lock.yml index f7d734b07ab..8071fb733e2 100644 --- a/.github/workflows/api-consumption-report.lock.yml +++ b/.github/workflows/api-consumption-report.lock.yml @@ -1589,6 +1589,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1602,6 +1775,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/approach-validator.lock.yml b/.github/workflows/approach-validator.lock.yml index 02f05e4ea46..0f44031a94c 100644 --- a/.github/workflows/approach-validator.lock.yml +++ b/.github/workflows/approach-validator.lock.yml @@ -1293,6 +1293,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1306,6 +1479,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/archie.lock.yml b/.github/workflows/archie.lock.yml index 15611d6af22..2b332d9bf81 100644 --- a/.github/workflows/archie.lock.yml +++ b/.github/workflows/archie.lock.yml @@ -1178,6 +1178,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1191,6 +1364,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/architecture-guardian.lock.yml b/.github/workflows/architecture-guardian.lock.yml index beae5130014..4c9283ce36e 100644 --- a/.github/workflows/architecture-guardian.lock.yml +++ b/.github/workflows/architecture-guardian.lock.yml @@ -1186,6 +1186,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1199,6 +1372,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/artifacts-summary.lock.yml b/.github/workflows/artifacts-summary.lock.yml index 4e3527ceb33..d4bce9a856b 100644 --- a/.github/workflows/artifacts-summary.lock.yml +++ b/.github/workflows/artifacts-summary.lock.yml @@ -1092,6 +1092,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1105,6 +1278,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/audit-workflows.lock.yml b/.github/workflows/audit-workflows.lock.yml index b3dc03c52f0..b2545e74d19 100644 --- a/.github/workflows/audit-workflows.lock.yml +++ b/.github/workflows/audit-workflows.lock.yml @@ -1386,6 +1386,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1399,6 +1572,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/auto-triage-issues.lock.yml b/.github/workflows/auto-triage-issues.lock.yml index f3f4d179cc8..a584a17506a 100644 --- a/.github/workflows/auto-triage-issues.lock.yml +++ b/.github/workflows/auto-triage-issues.lock.yml @@ -1119,6 +1119,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1132,6 +1305,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/avenger.lock.yml b/.github/workflows/avenger.lock.yml index 2023640a6c9..b16468c7c83 100644 --- a/.github/workflows/avenger.lock.yml +++ b/.github/workflows/avenger.lock.yml @@ -1276,6 +1276,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1289,6 +1462,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/aw-failure-investigator.lock.yml b/.github/workflows/aw-failure-investigator.lock.yml index 370cc5bf56f..d91a7470658 100644 --- a/.github/workflows/aw-failure-investigator.lock.yml +++ b/.github/workflows/aw-failure-investigator.lock.yml @@ -1377,6 +1377,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1390,6 +1563,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/blog-auditor.lock.yml b/.github/workflows/blog-auditor.lock.yml index 434eec01ae0..2bd9b4f43ad 100644 --- a/.github/workflows/blog-auditor.lock.yml +++ b/.github/workflows/blog-auditor.lock.yml @@ -1255,6 +1255,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1268,6 +1441,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/bot-detection.lock.yml b/.github/workflows/bot-detection.lock.yml index dcc9ad23b24..af64afd1c66 100644 --- a/.github/workflows/bot-detection.lock.yml +++ b/.github/workflows/bot-detection.lock.yml @@ -1180,6 +1180,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1193,6 +1366,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/brave.lock.yml b/.github/workflows/brave.lock.yml index 72e6a02b8e2..30f21ebabbf 100644 --- a/.github/workflows/brave.lock.yml +++ b/.github/workflows/brave.lock.yml @@ -1174,6 +1174,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1187,6 +1360,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/breaking-change-checker.lock.yml b/.github/workflows/breaking-change-checker.lock.yml index 104c38356a5..a5aa44c973c 100644 --- a/.github/workflows/breaking-change-checker.lock.yml +++ b/.github/workflows/breaking-change-checker.lock.yml @@ -1134,6 +1134,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1147,6 +1320,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/changeset.lock.yml b/.github/workflows/changeset.lock.yml index 38c146cdbae..5a2d8a940a5 100644 --- a/.github/workflows/changeset.lock.yml +++ b/.github/workflows/changeset.lock.yml @@ -1221,6 +1221,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1234,6 +1407,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/chaos-pr-bundle-fuzzer.lock.yml b/.github/workflows/chaos-pr-bundle-fuzzer.lock.yml index e3b60fdd088..7583cfbe7e5 100644 --- a/.github/workflows/chaos-pr-bundle-fuzzer.lock.yml +++ b/.github/workflows/chaos-pr-bundle-fuzzer.lock.yml @@ -1118,6 +1118,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1131,6 +1304,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/ci-coach.lock.yml b/.github/workflows/ci-coach.lock.yml index f6057496f83..1cdcd80e6d0 100644 --- a/.github/workflows/ci-coach.lock.yml +++ b/.github/workflows/ci-coach.lock.yml @@ -1228,6 +1228,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1241,6 +1414,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/ci-doctor.lock.yml b/.github/workflows/ci-doctor.lock.yml index d1e0f589bcc..e86163dbb52 100644 --- a/.github/workflows/ci-doctor.lock.yml +++ b/.github/workflows/ci-doctor.lock.yml @@ -1397,6 +1397,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1410,6 +1583,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/claude-code-user-docs-review.lock.yml b/.github/workflows/claude-code-user-docs-review.lock.yml index 38c9e0e4a21..237c04d3011 100644 --- a/.github/workflows/claude-code-user-docs-review.lock.yml +++ b/.github/workflows/claude-code-user-docs-review.lock.yml @@ -1223,6 +1223,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1236,6 +1409,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/cli-consistency-checker.lock.yml b/.github/workflows/cli-consistency-checker.lock.yml index 3f52e9f4cbf..4d760ec33c0 100644 --- a/.github/workflows/cli-consistency-checker.lock.yml +++ b/.github/workflows/cli-consistency-checker.lock.yml @@ -1107,6 +1107,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1120,6 +1293,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/cli-version-checker.lock.yml b/.github/workflows/cli-version-checker.lock.yml index e1893c815d0..2892c8153e8 100644 --- a/.github/workflows/cli-version-checker.lock.yml +++ b/.github/workflows/cli-version-checker.lock.yml @@ -1217,6 +1217,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1230,6 +1403,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/cloclo.lock.yml b/.github/workflows/cloclo.lock.yml index 82db25679f7..055ea17cdf7 100644 --- a/.github/workflows/cloclo.lock.yml +++ b/.github/workflows/cloclo.lock.yml @@ -1504,6 +1504,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1517,6 +1690,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/code-scanning-fixer.lock.yml b/.github/workflows/code-scanning-fixer.lock.yml index e6f6fc601e8..5699a9d0740 100644 --- a/.github/workflows/code-scanning-fixer.lock.yml +++ b/.github/workflows/code-scanning-fixer.lock.yml @@ -1215,6 +1215,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1228,6 +1401,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/code-simplifier.lock.yml b/.github/workflows/code-simplifier.lock.yml index 7075316ce8a..9969893d902 100644 --- a/.github/workflows/code-simplifier.lock.yml +++ b/.github/workflows/code-simplifier.lock.yml @@ -1171,6 +1171,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1184,6 +1357,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/codex-github-remote-mcp-test.lock.yml b/.github/workflows/codex-github-remote-mcp-test.lock.yml index b4db05f5229..2555ec70149 100644 --- a/.github/workflows/codex-github-remote-mcp-test.lock.yml +++ b/.github/workflows/codex-github-remote-mcp-test.lock.yml @@ -1055,6 +1055,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1068,6 +1241,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/commit-changes-analyzer.lock.yml b/.github/workflows/commit-changes-analyzer.lock.yml index 41f24c9d2fb..420a49aa34b 100644 --- a/.github/workflows/commit-changes-analyzer.lock.yml +++ b/.github/workflows/commit-changes-analyzer.lock.yml @@ -1063,6 +1063,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1076,6 +1249,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/constraint-solving-potd.lock.yml b/.github/workflows/constraint-solving-potd.lock.yml index 5b7ce6fcea5..587a99c4adf 100644 --- a/.github/workflows/constraint-solving-potd.lock.yml +++ b/.github/workflows/constraint-solving-potd.lock.yml @@ -1117,6 +1117,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1130,6 +1303,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/contribution-check.lock.yml b/.github/workflows/contribution-check.lock.yml index 8430056af68..699509c85bb 100644 --- a/.github/workflows/contribution-check.lock.yml +++ b/.github/workflows/contribution-check.lock.yml @@ -1230,6 +1230,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1243,6 +1416,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/copilot-agent-analysis.lock.yml b/.github/workflows/copilot-agent-analysis.lock.yml index c5b8d06d17e..3f99cd0afcd 100644 --- a/.github/workflows/copilot-agent-analysis.lock.yml +++ b/.github/workflows/copilot-agent-analysis.lock.yml @@ -1300,6 +1300,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1313,6 +1486,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/copilot-centralization-drilldown.lock.yml b/.github/workflows/copilot-centralization-drilldown.lock.yml index 4002d1e2a74..2902f3e9bf3 100644 --- a/.github/workflows/copilot-centralization-drilldown.lock.yml +++ b/.github/workflows/copilot-centralization-drilldown.lock.yml @@ -1080,6 +1080,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1093,6 +1266,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/copilot-centralization-optimizer.lock.yml b/.github/workflows/copilot-centralization-optimizer.lock.yml index e332643aa53..86f94f9129e 100644 --- a/.github/workflows/copilot-centralization-optimizer.lock.yml +++ b/.github/workflows/copilot-centralization-optimizer.lock.yml @@ -1128,6 +1128,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1141,6 +1314,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/copilot-cli-deep-research.lock.yml b/.github/workflows/copilot-cli-deep-research.lock.yml index 0daacd9625a..686708f88b6 100644 --- a/.github/workflows/copilot-cli-deep-research.lock.yml +++ b/.github/workflows/copilot-cli-deep-research.lock.yml @@ -1135,6 +1135,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1148,6 +1321,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/copilot-opt.lock.yml b/.github/workflows/copilot-opt.lock.yml index 9024bf5ac74..1abe808a851 100644 --- a/.github/workflows/copilot-opt.lock.yml +++ b/.github/workflows/copilot-opt.lock.yml @@ -1205,6 +1205,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1218,6 +1391,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/copilot-pr-merged-report.lock.yml b/.github/workflows/copilot-pr-merged-report.lock.yml index 6c2cf47fa03..45edabe1356 100644 --- a/.github/workflows/copilot-pr-merged-report.lock.yml +++ b/.github/workflows/copilot-pr-merged-report.lock.yml @@ -1073,6 +1073,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1086,6 +1259,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/copilot-pr-nlp-analysis.lock.yml b/.github/workflows/copilot-pr-nlp-analysis.lock.yml index f41b3f10533..2188e2531fd 100644 --- a/.github/workflows/copilot-pr-nlp-analysis.lock.yml +++ b/.github/workflows/copilot-pr-nlp-analysis.lock.yml @@ -1261,6 +1261,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1274,6 +1447,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/copilot-pr-prompt-analysis.lock.yml b/.github/workflows/copilot-pr-prompt-analysis.lock.yml index eef523ca0d2..2ee4fe03a56 100644 --- a/.github/workflows/copilot-pr-prompt-analysis.lock.yml +++ b/.github/workflows/copilot-pr-prompt-analysis.lock.yml @@ -1200,6 +1200,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1213,6 +1386,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/copilot-session-insights.lock.yml b/.github/workflows/copilot-session-insights.lock.yml index adaa31e1ec0..1666a20c518 100644 --- a/.github/workflows/copilot-session-insights.lock.yml +++ b/.github/workflows/copilot-session-insights.lock.yml @@ -1319,6 +1319,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1332,6 +1505,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/craft.lock.yml b/.github/workflows/craft.lock.yml index 0aa368a4476..e1c64216975 100644 --- a/.github/workflows/craft.lock.yml +++ b/.github/workflows/craft.lock.yml @@ -1175,6 +1175,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1188,6 +1361,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-agent-of-the-day-blog-writer.lock.yml b/.github/workflows/daily-agent-of-the-day-blog-writer.lock.yml index f2a5e64ecde..a338cc3cef9 100644 --- a/.github/workflows/daily-agent-of-the-day-blog-writer.lock.yml +++ b/.github/workflows/daily-agent-of-the-day-blog-writer.lock.yml @@ -1308,6 +1308,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1321,6 +1494,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-agentrx-trace-optimizer.lock.yml b/.github/workflows/daily-agentrx-trace-optimizer.lock.yml index 2cd14ad7858..5f73655a085 100644 --- a/.github/workflows/daily-agentrx-trace-optimizer.lock.yml +++ b/.github/workflows/daily-agentrx-trace-optimizer.lock.yml @@ -1324,6 +1324,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1337,6 +1510,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-ambient-context-optimizer.lock.yml b/.github/workflows/daily-ambient-context-optimizer.lock.yml index 29a539d8fe4..8849f0ce08d 100644 --- a/.github/workflows/daily-ambient-context-optimizer.lock.yml +++ b/.github/workflows/daily-ambient-context-optimizer.lock.yml @@ -1193,6 +1193,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1206,6 +1379,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-architecture-diagram.lock.yml b/.github/workflows/daily-architecture-diagram.lock.yml index e8024b65b2c..af59e3c7939 100644 --- a/.github/workflows/daily-architecture-diagram.lock.yml +++ b/.github/workflows/daily-architecture-diagram.lock.yml @@ -1265,6 +1265,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1278,6 +1451,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-assign-issue-to-user.lock.yml b/.github/workflows/daily-assign-issue-to-user.lock.yml index ed9e7972854..7ba5255e15a 100644 --- a/.github/workflows/daily-assign-issue-to-user.lock.yml +++ b/.github/workflows/daily-assign-issue-to-user.lock.yml @@ -1104,6 +1104,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1117,6 +1290,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-astrostylelite-markdown-spellcheck.lock.yml b/.github/workflows/daily-astrostylelite-markdown-spellcheck.lock.yml index 930e8155b3d..a28db65c39a 100644 --- a/.github/workflows/daily-astrostylelite-markdown-spellcheck.lock.yml +++ b/.github/workflows/daily-astrostylelite-markdown-spellcheck.lock.yml @@ -1218,6 +1218,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1231,6 +1404,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-aw-cross-repo-compile-check.lock.yml b/.github/workflows/daily-aw-cross-repo-compile-check.lock.yml index f0c5631c285..5eb237bafd5 100644 --- a/.github/workflows/daily-aw-cross-repo-compile-check.lock.yml +++ b/.github/workflows/daily-aw-cross-repo-compile-check.lock.yml @@ -1213,6 +1213,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1226,6 +1399,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-awf-spec-compiler-surfacing.lock.yml b/.github/workflows/daily-awf-spec-compiler-surfacing.lock.yml index c4d163f7763..e6d342602fc 100644 --- a/.github/workflows/daily-awf-spec-compiler-surfacing.lock.yml +++ b/.github/workflows/daily-awf-spec-compiler-surfacing.lock.yml @@ -1106,6 +1106,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1119,6 +1292,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-byok-ollama-test.lock.yml b/.github/workflows/daily-byok-ollama-test.lock.yml index d6ad2ad4496..0d30cf05b5c 100644 --- a/.github/workflows/daily-byok-ollama-test.lock.yml +++ b/.github/workflows/daily-byok-ollama-test.lock.yml @@ -1083,6 +1083,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1096,6 +1269,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-cache-strategy-analyzer.lock.yml b/.github/workflows/daily-cache-strategy-analyzer.lock.yml index 9b643209a04..a137774dfd4 100644 --- a/.github/workflows/daily-cache-strategy-analyzer.lock.yml +++ b/.github/workflows/daily-cache-strategy-analyzer.lock.yml @@ -1351,6 +1351,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1364,6 +1537,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-caveman-optimizer.lock.yml b/.github/workflows/daily-caveman-optimizer.lock.yml index f57ff254876..a2d54fc0c0c 100644 --- a/.github/workflows/daily-caveman-optimizer.lock.yml +++ b/.github/workflows/daily-caveman-optimizer.lock.yml @@ -1256,6 +1256,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1269,6 +1442,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-choice-test.lock.yml b/.github/workflows/daily-choice-test.lock.yml index 8f1cbf04529..b502119cc83 100644 --- a/.github/workflows/daily-choice-test.lock.yml +++ b/.github/workflows/daily-choice-test.lock.yml @@ -1151,6 +1151,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1164,6 +1337,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-cli-performance.lock.yml b/.github/workflows/daily-cli-performance.lock.yml index a570578abe5..20f75d51c16 100644 --- a/.github/workflows/daily-cli-performance.lock.yml +++ b/.github/workflows/daily-cli-performance.lock.yml @@ -1387,6 +1387,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1400,6 +1573,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-cli-tools-tester.lock.yml b/.github/workflows/daily-cli-tools-tester.lock.yml index 81b5ef4e106..4ef4e0ea92c 100644 --- a/.github/workflows/daily-cli-tools-tester.lock.yml +++ b/.github/workflows/daily-cli-tools-tester.lock.yml @@ -1218,6 +1218,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1231,6 +1404,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-code-metrics.lock.yml b/.github/workflows/daily-code-metrics.lock.yml index 09399113e37..fee72eef5e3 100644 --- a/.github/workflows/daily-code-metrics.lock.yml +++ b/.github/workflows/daily-code-metrics.lock.yml @@ -1337,6 +1337,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1350,6 +1523,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-community-attribution.lock.yml b/.github/workflows/daily-community-attribution.lock.yml index 4344d9e3772..759ade6db0b 100644 --- a/.github/workflows/daily-community-attribution.lock.yml +++ b/.github/workflows/daily-community-attribution.lock.yml @@ -1277,6 +1277,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1290,6 +1463,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-compiler-quality.lock.yml b/.github/workflows/daily-compiler-quality.lock.yml index 5c4c18f5436..60454581a48 100644 --- a/.github/workflows/daily-compiler-quality.lock.yml +++ b/.github/workflows/daily-compiler-quality.lock.yml @@ -1253,6 +1253,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1266,6 +1439,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-compiler-threat-spec-optimizer.lock.yml b/.github/workflows/daily-compiler-threat-spec-optimizer.lock.yml index 8a7806d613b..4add5be50bc 100644 --- a/.github/workflows/daily-compiler-threat-spec-optimizer.lock.yml +++ b/.github/workflows/daily-compiler-threat-spec-optimizer.lock.yml @@ -1178,6 +1178,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1191,6 +1364,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-credit-limit-test.lock.yml b/.github/workflows/daily-credit-limit-test.lock.yml index f7bd5f49f3c..d640527ea82 100644 --- a/.github/workflows/daily-credit-limit-test.lock.yml +++ b/.github/workflows/daily-credit-limit-test.lock.yml @@ -1061,6 +1061,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1074,6 +1247,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-doc-healer.lock.yml b/.github/workflows/daily-doc-healer.lock.yml index 9c219beb65c..de4484a1f8b 100644 --- a/.github/workflows/daily-doc-healer.lock.yml +++ b/.github/workflows/daily-doc-healer.lock.yml @@ -1360,6 +1360,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1373,6 +1546,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-doc-updater.lock.yml b/.github/workflows/daily-doc-updater.lock.yml index b5f8dbf6cd7..51c6693affc 100644 --- a/.github/workflows/daily-doc-updater.lock.yml +++ b/.github/workflows/daily-doc-updater.lock.yml @@ -1162,6 +1162,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1175,6 +1348,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-experiment-report.lock.yml b/.github/workflows/daily-experiment-report.lock.yml index d2ec096971c..b7240b599d3 100644 --- a/.github/workflows/daily-experiment-report.lock.yml +++ b/.github/workflows/daily-experiment-report.lock.yml @@ -1249,6 +1249,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1262,6 +1435,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-fact.lock.yml b/.github/workflows/daily-fact.lock.yml index 4b258b71aeb..f8f86031edd 100644 --- a/.github/workflows/daily-fact.lock.yml +++ b/.github/workflows/daily-fact.lock.yml @@ -1364,6 +1364,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1377,6 +1550,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-file-diet.lock.yml b/.github/workflows/daily-file-diet.lock.yml index 44d00debce2..d2de890c660 100644 --- a/.github/workflows/daily-file-diet.lock.yml +++ b/.github/workflows/daily-file-diet.lock.yml @@ -1175,6 +1175,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1188,6 +1361,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-firewall-report.lock.yml b/.github/workflows/daily-firewall-report.lock.yml index c0b2357078c..b691f538e54 100644 --- a/.github/workflows/daily-firewall-report.lock.yml +++ b/.github/workflows/daily-firewall-report.lock.yml @@ -1177,6 +1177,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1190,6 +1363,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-formal-spec-verifier.lock.yml b/.github/workflows/daily-formal-spec-verifier.lock.yml index eadaf586155..18a3de2ab90 100644 --- a/.github/workflows/daily-formal-spec-verifier.lock.yml +++ b/.github/workflows/daily-formal-spec-verifier.lock.yml @@ -1221,6 +1221,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1234,6 +1407,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-function-namer.lock.yml b/.github/workflows/daily-function-namer.lock.yml index c64a48bf88c..6fcaa3dd63b 100644 --- a/.github/workflows/daily-function-namer.lock.yml +++ b/.github/workflows/daily-function-namer.lock.yml @@ -1178,6 +1178,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1191,6 +1364,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-geo-optimizer.lock.yml b/.github/workflows/daily-geo-optimizer.lock.yml index 257cb120c83..934a2e1ab53 100644 --- a/.github/workflows/daily-geo-optimizer.lock.yml +++ b/.github/workflows/daily-geo-optimizer.lock.yml @@ -1129,6 +1129,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1142,6 +1315,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-hippo-learn.lock.yml b/.github/workflows/daily-hippo-learn.lock.yml index 583326e793f..f40e8ae3d5f 100644 --- a/.github/workflows/daily-hippo-learn.lock.yml +++ b/.github/workflows/daily-hippo-learn.lock.yml @@ -1232,6 +1232,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1245,6 +1418,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-issues-report.lock.yml b/.github/workflows/daily-issues-report.lock.yml index a560181de89..a2d50dec323 100644 --- a/.github/workflows/daily-issues-report.lock.yml +++ b/.github/workflows/daily-issues-report.lock.yml @@ -1404,6 +1404,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1417,6 +1590,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-malicious-code-scan.lock.yml b/.github/workflows/daily-malicious-code-scan.lock.yml index 8e6c8c975fe..9f390b787ac 100644 --- a/.github/workflows/daily-malicious-code-scan.lock.yml +++ b/.github/workflows/daily-malicious-code-scan.lock.yml @@ -1139,6 +1139,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1152,6 +1325,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-max-ai-credits-test.lock.yml b/.github/workflows/daily-max-ai-credits-test.lock.yml index 3ea77ab8a5c..0dbd21189e8 100644 --- a/.github/workflows/daily-max-ai-credits-test.lock.yml +++ b/.github/workflows/daily-max-ai-credits-test.lock.yml @@ -1000,6 +1000,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1013,6 +1186,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Process no-op messages id: noop diff --git a/.github/workflows/daily-mcp-concurrency-analysis.lock.yml b/.github/workflows/daily-mcp-concurrency-analysis.lock.yml index 9d3b60c6a90..82f3b7b9b6b 100644 --- a/.github/workflows/daily-mcp-concurrency-analysis.lock.yml +++ b/.github/workflows/daily-mcp-concurrency-analysis.lock.yml @@ -1257,6 +1257,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1270,6 +1443,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-model-inventory.lock.yml b/.github/workflows/daily-model-inventory.lock.yml index 5b88b1a694c..4264cd93501 100644 --- a/.github/workflows/daily-model-inventory.lock.yml +++ b/.github/workflows/daily-model-inventory.lock.yml @@ -1447,6 +1447,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1460,6 +1633,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-multi-device-docs-tester.lock.yml b/.github/workflows/daily-multi-device-docs-tester.lock.yml index b60b1bcf19c..34c844fe368 100644 --- a/.github/workflows/daily-multi-device-docs-tester.lock.yml +++ b/.github/workflows/daily-multi-device-docs-tester.lock.yml @@ -1152,6 +1152,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1165,6 +1338,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-news.lock.yml b/.github/workflows/daily-news.lock.yml index 152075ef16b..fa12f9bf386 100644 --- a/.github/workflows/daily-news.lock.yml +++ b/.github/workflows/daily-news.lock.yml @@ -1372,6 +1372,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1385,6 +1558,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-observability-report.lock.yml b/.github/workflows/daily-observability-report.lock.yml index 23c671da498..7d4d0ee7286 100644 --- a/.github/workflows/daily-observability-report.lock.yml +++ b/.github/workflows/daily-observability-report.lock.yml @@ -1223,6 +1223,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1236,6 +1409,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-performance-summary.lock.yml b/.github/workflows/daily-performance-summary.lock.yml index f200fc2f7b3..0aeb9434fe3 100644 --- a/.github/workflows/daily-performance-summary.lock.yml +++ b/.github/workflows/daily-performance-summary.lock.yml @@ -1687,6 +1687,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1700,6 +1873,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-regulatory.lock.yml b/.github/workflows/daily-regulatory.lock.yml index 760846e04a3..29e638edbe8 100644 --- a/.github/workflows/daily-regulatory.lock.yml +++ b/.github/workflows/daily-regulatory.lock.yml @@ -1616,6 +1616,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1629,6 +1802,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-reliability-review.lock.yml b/.github/workflows/daily-reliability-review.lock.yml index 9a6eeab2e5d..b74c53a70f5 100644 --- a/.github/workflows/daily-reliability-review.lock.yml +++ b/.github/workflows/daily-reliability-review.lock.yml @@ -1235,6 +1235,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1248,6 +1421,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-rendering-scripts-verifier.lock.yml b/.github/workflows/daily-rendering-scripts-verifier.lock.yml index 3b6e65b25dc..5a532703017 100644 --- a/.github/workflows/daily-rendering-scripts-verifier.lock.yml +++ b/.github/workflows/daily-rendering-scripts-verifier.lock.yml @@ -1388,6 +1388,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1401,6 +1574,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-repo-chronicle.lock.yml b/.github/workflows/daily-repo-chronicle.lock.yml index 3cb1d9b9e36..eafe925194d 100644 --- a/.github/workflows/daily-repo-chronicle.lock.yml +++ b/.github/workflows/daily-repo-chronicle.lock.yml @@ -1193,6 +1193,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1206,6 +1379,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-safe-output-integrator.lock.yml b/.github/workflows/daily-safe-output-integrator.lock.yml index c84647a1413..4b8d2137f47 100644 --- a/.github/workflows/daily-safe-output-integrator.lock.yml +++ b/.github/workflows/daily-safe-output-integrator.lock.yml @@ -1177,6 +1177,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1190,6 +1363,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-safe-output-optimizer.lock.yml b/.github/workflows/daily-safe-output-optimizer.lock.yml index 280e089c463..fabbe9e23d4 100644 --- a/.github/workflows/daily-safe-output-optimizer.lock.yml +++ b/.github/workflows/daily-safe-output-optimizer.lock.yml @@ -1409,6 +1409,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1422,6 +1595,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-safe-outputs-conformance.lock.yml b/.github/workflows/daily-safe-outputs-conformance.lock.yml index 521f1c67305..9c08aee2d3d 100644 --- a/.github/workflows/daily-safe-outputs-conformance.lock.yml +++ b/.github/workflows/daily-safe-outputs-conformance.lock.yml @@ -1191,6 +1191,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1204,6 +1377,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-safeoutputs-git-simulator.lock.yml b/.github/workflows/daily-safeoutputs-git-simulator.lock.yml index 97ea01d2d79..409a92a4bf8 100644 --- a/.github/workflows/daily-safeoutputs-git-simulator.lock.yml +++ b/.github/workflows/daily-safeoutputs-git-simulator.lock.yml @@ -1250,6 +1250,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1263,6 +1436,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-secrets-analysis.lock.yml b/.github/workflows/daily-secrets-analysis.lock.yml index 37f6820ae84..fa4a927b8df 100644 --- a/.github/workflows/daily-secrets-analysis.lock.yml +++ b/.github/workflows/daily-secrets-analysis.lock.yml @@ -1095,6 +1095,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1108,6 +1281,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-security-observability.lock.yml b/.github/workflows/daily-security-observability.lock.yml index def4dd97e39..9b944853945 100644 --- a/.github/workflows/daily-security-observability.lock.yml +++ b/.github/workflows/daily-security-observability.lock.yml @@ -1319,6 +1319,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1332,6 +1505,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-security-red-team.lock.yml b/.github/workflows/daily-security-red-team.lock.yml index 765c6490f0a..2e92622855e 100644 --- a/.github/workflows/daily-security-red-team.lock.yml +++ b/.github/workflows/daily-security-red-team.lock.yml @@ -1288,6 +1288,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1301,6 +1474,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-semgrep-scan.lock.yml b/.github/workflows/daily-semgrep-scan.lock.yml index e20a3f0db26..8cca628e685 100644 --- a/.github/workflows/daily-semgrep-scan.lock.yml +++ b/.github/workflows/daily-semgrep-scan.lock.yml @@ -1175,6 +1175,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1188,6 +1361,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-sentrux-report.lock.yml b/.github/workflows/daily-sentrux-report.lock.yml index 3165c23f4e2..ea97dddc3da 100644 --- a/.github/workflows/daily-sentrux-report.lock.yml +++ b/.github/workflows/daily-sentrux-report.lock.yml @@ -1152,6 +1152,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1165,6 +1338,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-skill-optimizer.lock.yml b/.github/workflows/daily-skill-optimizer.lock.yml index 3a6228af961..8d9bf922b77 100644 --- a/.github/workflows/daily-skill-optimizer.lock.yml +++ b/.github/workflows/daily-skill-optimizer.lock.yml @@ -1118,6 +1118,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1131,6 +1304,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-spdd-spec-planner.lock.yml b/.github/workflows/daily-spdd-spec-planner.lock.yml index ea471cccc30..e23e9e0b235 100644 --- a/.github/workflows/daily-spdd-spec-planner.lock.yml +++ b/.github/workflows/daily-spdd-spec-planner.lock.yml @@ -1180,6 +1180,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1193,6 +1366,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-syntax-error-quality.lock.yml b/.github/workflows/daily-syntax-error-quality.lock.yml index 915f62dd497..5ef83fbd122 100644 --- a/.github/workflows/daily-syntax-error-quality.lock.yml +++ b/.github/workflows/daily-syntax-error-quality.lock.yml @@ -1119,6 +1119,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1132,6 +1305,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-team-evolution-insights.lock.yml b/.github/workflows/daily-team-evolution-insights.lock.yml index c20394795d8..06dda77df87 100644 --- a/.github/workflows/daily-team-evolution-insights.lock.yml +++ b/.github/workflows/daily-team-evolution-insights.lock.yml @@ -1160,6 +1160,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1173,6 +1346,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-team-status.lock.yml b/.github/workflows/daily-team-status.lock.yml index 9abc731d401..cc3d88947ef 100644 --- a/.github/workflows/daily-team-status.lock.yml +++ b/.github/workflows/daily-team-status.lock.yml @@ -1076,6 +1076,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1089,6 +1262,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-testify-uber-super-expert.lock.yml b/.github/workflows/daily-testify-uber-super-expert.lock.yml index e2ce194035e..4289a44ff19 100644 --- a/.github/workflows/daily-testify-uber-super-expert.lock.yml +++ b/.github/workflows/daily-testify-uber-super-expert.lock.yml @@ -1224,6 +1224,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1237,6 +1410,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-token-consumption-report.lock.yml b/.github/workflows/daily-token-consumption-report.lock.yml index b177ddb61cf..35157234f55 100644 --- a/.github/workflows/daily-token-consumption-report.lock.yml +++ b/.github/workflows/daily-token-consumption-report.lock.yml @@ -1314,6 +1314,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1327,6 +1500,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-windows-terminal-integration-builder.lock.yml b/.github/workflows/daily-windows-terminal-integration-builder.lock.yml index 933b3c70d18..72186634285 100644 --- a/.github/workflows/daily-windows-terminal-integration-builder.lock.yml +++ b/.github/workflows/daily-windows-terminal-integration-builder.lock.yml @@ -1059,6 +1059,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1072,6 +1245,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/daily-workflow-updater.lock.yml b/.github/workflows/daily-workflow-updater.lock.yml index ad4e7e08d6f..8dd576b3f50 100644 --- a/.github/workflows/daily-workflow-updater.lock.yml +++ b/.github/workflows/daily-workflow-updater.lock.yml @@ -1106,6 +1106,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1119,6 +1292,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/dataflow-pr-discussion-dataset.lock.yml b/.github/workflows/dataflow-pr-discussion-dataset.lock.yml index 7084377103f..a38a7f6d458 100644 --- a/.github/workflows/dataflow-pr-discussion-dataset.lock.yml +++ b/.github/workflows/dataflow-pr-discussion-dataset.lock.yml @@ -1468,6 +1468,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1481,6 +1654,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/dead-code-remover.lock.yml b/.github/workflows/dead-code-remover.lock.yml index 5f097f6fe30..609312d6185 100644 --- a/.github/workflows/dead-code-remover.lock.yml +++ b/.github/workflows/dead-code-remover.lock.yml @@ -1178,6 +1178,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1191,6 +1364,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/deep-report.lock.yml b/.github/workflows/deep-report.lock.yml index ce20ca91022..bbb405d5b6d 100644 --- a/.github/workflows/deep-report.lock.yml +++ b/.github/workflows/deep-report.lock.yml @@ -1660,6 +1660,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1673,6 +1846,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/delight.lock.yml b/.github/workflows/delight.lock.yml index 81a41e8a55d..97b4f4d1464 100644 --- a/.github/workflows/delight.lock.yml +++ b/.github/workflows/delight.lock.yml @@ -1207,6 +1207,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1220,6 +1393,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/dependabot-burner.lock.yml b/.github/workflows/dependabot-burner.lock.yml index 635c56be907..95dfc93c8f3 100644 --- a/.github/workflows/dependabot-burner.lock.yml +++ b/.github/workflows/dependabot-burner.lock.yml @@ -1250,6 +1250,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1263,6 +1436,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/dependabot-go-checker.lock.yml b/.github/workflows/dependabot-go-checker.lock.yml index bc119776e59..604378905de 100644 --- a/.github/workflows/dependabot-go-checker.lock.yml +++ b/.github/workflows/dependabot-go-checker.lock.yml @@ -1166,6 +1166,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1179,6 +1352,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/dependabot-repair.lock.yml b/.github/workflows/dependabot-repair.lock.yml index c859559bbe2..3a98a8a038a 100644 --- a/.github/workflows/dependabot-repair.lock.yml +++ b/.github/workflows/dependabot-repair.lock.yml @@ -1206,6 +1206,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1219,6 +1392,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/deployment-incident-monitor.lock.yml b/.github/workflows/deployment-incident-monitor.lock.yml index afd9adbd2ee..abc76efff80 100644 --- a/.github/workflows/deployment-incident-monitor.lock.yml +++ b/.github/workflows/deployment-incident-monitor.lock.yml @@ -1116,6 +1116,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1129,6 +1302,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/design-decision-gate.lock.yml b/.github/workflows/design-decision-gate.lock.yml index 93b58a59858..f67657911bf 100644 --- a/.github/workflows/design-decision-gate.lock.yml +++ b/.github/workflows/design-decision-gate.lock.yml @@ -1301,6 +1301,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1314,6 +1487,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/designer-drift-audit.lock.yml b/.github/workflows/designer-drift-audit.lock.yml index 4812d1f3923..61e8464ff6c 100644 --- a/.github/workflows/designer-drift-audit.lock.yml +++ b/.github/workflows/designer-drift-audit.lock.yml @@ -1065,6 +1065,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1078,6 +1251,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/dev-hawk.lock.yml b/.github/workflows/dev-hawk.lock.yml index 3365117ecd5..6e4ea77eb49 100644 --- a/.github/workflows/dev-hawk.lock.yml +++ b/.github/workflows/dev-hawk.lock.yml @@ -1224,6 +1224,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1237,6 +1410,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/dev.lock.yml b/.github/workflows/dev.lock.yml index d76ad58dcdc..cf732f9fb54 100644 --- a/.github/workflows/dev.lock.yml +++ b/.github/workflows/dev.lock.yml @@ -1184,6 +1184,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1197,6 +1370,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/developer-docs-consolidator.lock.yml b/.github/workflows/developer-docs-consolidator.lock.yml index f8af01a6b38..8460f519437 100644 --- a/.github/workflows/developer-docs-consolidator.lock.yml +++ b/.github/workflows/developer-docs-consolidator.lock.yml @@ -1357,6 +1357,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1370,6 +1543,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/dictation-prompt.lock.yml b/.github/workflows/dictation-prompt.lock.yml index 1c5ae1ceee5..bbdea75520b 100644 --- a/.github/workflows/dictation-prompt.lock.yml +++ b/.github/workflows/dictation-prompt.lock.yml @@ -1108,6 +1108,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1121,6 +1294,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/discussion-task-miner.lock.yml b/.github/workflows/discussion-task-miner.lock.yml index 9346244ee7e..bdcbda2efa5 100644 --- a/.github/workflows/discussion-task-miner.lock.yml +++ b/.github/workflows/discussion-task-miner.lock.yml @@ -1190,6 +1190,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1203,6 +1376,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/docs-noob-tester.lock.yml b/.github/workflows/docs-noob-tester.lock.yml index 3c4c8e5fdc6..800f7c73f3a 100644 --- a/.github/workflows/docs-noob-tester.lock.yml +++ b/.github/workflows/docs-noob-tester.lock.yml @@ -1160,6 +1160,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1173,6 +1346,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/draft-pr-cleanup.lock.yml b/.github/workflows/draft-pr-cleanup.lock.yml index 8e5c7871380..d78ce421fe8 100644 --- a/.github/workflows/draft-pr-cleanup.lock.yml +++ b/.github/workflows/draft-pr-cleanup.lock.yml @@ -1142,6 +1142,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1155,6 +1328,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/duplicate-code-detector.lock.yml b/.github/workflows/duplicate-code-detector.lock.yml index fc5d7b79138..6312ea4f36d 100644 --- a/.github/workflows/duplicate-code-detector.lock.yml +++ b/.github/workflows/duplicate-code-detector.lock.yml @@ -1201,6 +1201,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1214,6 +1387,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/example-failure-category-filter.lock.yml b/.github/workflows/example-failure-category-filter.lock.yml index 71dd9bd5b32..de6a5c04e39 100644 --- a/.github/workflows/example-failure-category-filter.lock.yml +++ b/.github/workflows/example-failure-category-filter.lock.yml @@ -1053,6 +1053,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1066,6 +1239,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/example-permissions-warning.lock.yml b/.github/workflows/example-permissions-warning.lock.yml index 1dfab3df8b0..e5df06d4aec 100644 --- a/.github/workflows/example-permissions-warning.lock.yml +++ b/.github/workflows/example-permissions-warning.lock.yml @@ -1016,6 +1016,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1029,6 +1202,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/example-workflow-analyzer.lock.yml b/.github/workflows/example-workflow-analyzer.lock.yml index 22be71418ab..57b5f71813d 100644 --- a/.github/workflows/example-workflow-analyzer.lock.yml +++ b/.github/workflows/example-workflow-analyzer.lock.yml @@ -1240,6 +1240,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1253,6 +1426,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/firewall-escape.lock.yml b/.github/workflows/firewall-escape.lock.yml index 53d906739be..f6ed58eed67 100644 --- a/.github/workflows/firewall-escape.lock.yml +++ b/.github/workflows/firewall-escape.lock.yml @@ -1201,6 +1201,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1214,6 +1387,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/firewall.lock.yml b/.github/workflows/firewall.lock.yml index a3d08840ae7..1d11dc16e64 100644 --- a/.github/workflows/firewall.lock.yml +++ b/.github/workflows/firewall.lock.yml @@ -1024,6 +1024,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1037,6 +1210,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/functional-pragmatist.lock.yml b/.github/workflows/functional-pragmatist.lock.yml index 06125ffdc0c..909c083b5e0 100644 --- a/.github/workflows/functional-pragmatist.lock.yml +++ b/.github/workflows/functional-pragmatist.lock.yml @@ -1114,6 +1114,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1127,6 +1300,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/github-mcp-structural-analysis.lock.yml b/.github/workflows/github-mcp-structural-analysis.lock.yml index 8826e82a14d..9341acae493 100644 --- a/.github/workflows/github-mcp-structural-analysis.lock.yml +++ b/.github/workflows/github-mcp-structural-analysis.lock.yml @@ -1263,6 +1263,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1276,6 +1449,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/github-mcp-tools-report.lock.yml b/.github/workflows/github-mcp-tools-report.lock.yml index a2d8697457f..9ef76228e2b 100644 --- a/.github/workflows/github-mcp-tools-report.lock.yml +++ b/.github/workflows/github-mcp-tools-report.lock.yml @@ -1254,6 +1254,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1267,6 +1440,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/github-remote-mcp-auth-test.lock.yml b/.github/workflows/github-remote-mcp-auth-test.lock.yml index 02587f602ea..a5e48996683 100644 --- a/.github/workflows/github-remote-mcp-auth-test.lock.yml +++ b/.github/workflows/github-remote-mcp-auth-test.lock.yml @@ -1110,6 +1110,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1123,6 +1296,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/glossary-maintainer.lock.yml b/.github/workflows/glossary-maintainer.lock.yml index f0b9efed121..9ac3629e54f 100644 --- a/.github/workflows/glossary-maintainer.lock.yml +++ b/.github/workflows/glossary-maintainer.lock.yml @@ -1258,6 +1258,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1271,6 +1444,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/go-fan.lock.yml b/.github/workflows/go-fan.lock.yml index 2e82f926455..e12c9837dfc 100644 --- a/.github/workflows/go-fan.lock.yml +++ b/.github/workflows/go-fan.lock.yml @@ -1286,6 +1286,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1299,6 +1472,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/go-logger.lock.yml b/.github/workflows/go-logger.lock.yml index 9b7cea698c5..cf48dd76012 100644 --- a/.github/workflows/go-logger.lock.yml +++ b/.github/workflows/go-logger.lock.yml @@ -1270,6 +1270,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1283,6 +1456,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/go-pattern-detector.lock.yml b/.github/workflows/go-pattern-detector.lock.yml index c5e8ce5b758..371f3dfee7d 100644 --- a/.github/workflows/go-pattern-detector.lock.yml +++ b/.github/workflows/go-pattern-detector.lock.yml @@ -1234,6 +1234,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1247,6 +1420,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/gpclean.lock.yml b/.github/workflows/gpclean.lock.yml index b91a482061e..b9da4a893ee 100644 --- a/.github/workflows/gpclean.lock.yml +++ b/.github/workflows/gpclean.lock.yml @@ -1195,6 +1195,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1208,6 +1381,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/grumpy-reviewer.lock.yml b/.github/workflows/grumpy-reviewer.lock.yml index d34ba9fa042..05bc6475cd2 100644 --- a/.github/workflows/grumpy-reviewer.lock.yml +++ b/.github/workflows/grumpy-reviewer.lock.yml @@ -1236,6 +1236,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1249,6 +1422,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/hippo-embed.lock.yml b/.github/workflows/hippo-embed.lock.yml index 6ea67abe785..96e6f0565d3 100644 --- a/.github/workflows/hippo-embed.lock.yml +++ b/.github/workflows/hippo-embed.lock.yml @@ -1146,6 +1146,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1159,6 +1332,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/hourly-ci-cleaner.lock.yml b/.github/workflows/hourly-ci-cleaner.lock.yml index ca70e64c886..cabba4b2f65 100644 --- a/.github/workflows/hourly-ci-cleaner.lock.yml +++ b/.github/workflows/hourly-ci-cleaner.lock.yml @@ -1271,6 +1271,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1284,6 +1457,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/instructions-janitor.lock.yml b/.github/workflows/instructions-janitor.lock.yml index 8d4ae4b52cc..c7d52e49b15 100644 --- a/.github/workflows/instructions-janitor.lock.yml +++ b/.github/workflows/instructions-janitor.lock.yml @@ -1245,6 +1245,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1258,6 +1431,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/issue-arborist.lock.yml b/.github/workflows/issue-arborist.lock.yml index fdf5df041bb..fc92dafdc22 100644 --- a/.github/workflows/issue-arborist.lock.yml +++ b/.github/workflows/issue-arborist.lock.yml @@ -1267,6 +1267,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1280,6 +1453,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/issue-monster.lock.yml b/.github/workflows/issue-monster.lock.yml index 38b33eaa4f9..6d0f5c08c76 100644 --- a/.github/workflows/issue-monster.lock.yml +++ b/.github/workflows/issue-monster.lock.yml @@ -1484,6 +1484,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1497,6 +1670,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/issue-triage-agent.lock.yml b/.github/workflows/issue-triage-agent.lock.yml index b2ebbb43347..10526a37be3 100644 --- a/.github/workflows/issue-triage-agent.lock.yml +++ b/.github/workflows/issue-triage-agent.lock.yml @@ -1090,6 +1090,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1103,6 +1276,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/jsweep.lock.yml b/.github/workflows/jsweep.lock.yml index 3cf14d0fae9..c70c90f198f 100644 --- a/.github/workflows/jsweep.lock.yml +++ b/.github/workflows/jsweep.lock.yml @@ -1166,6 +1166,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1179,6 +1352,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/layout-spec-maintainer.lock.yml b/.github/workflows/layout-spec-maintainer.lock.yml index 4cc91300024..80646f0f724 100644 --- a/.github/workflows/layout-spec-maintainer.lock.yml +++ b/.github/workflows/layout-spec-maintainer.lock.yml @@ -1154,6 +1154,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1167,6 +1340,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/lint-monster.lock.yml b/.github/workflows/lint-monster.lock.yml index c680df4f3cc..7f14018503f 100644 --- a/.github/workflows/lint-monster.lock.yml +++ b/.github/workflows/lint-monster.lock.yml @@ -1199,6 +1199,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1212,6 +1385,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/linter-miner.lock.yml b/.github/workflows/linter-miner.lock.yml index 18ef88f812d..5a64922e9d2 100644 --- a/.github/workflows/linter-miner.lock.yml +++ b/.github/workflows/linter-miner.lock.yml @@ -1196,6 +1196,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1209,6 +1382,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/lockfile-stats.lock.yml b/.github/workflows/lockfile-stats.lock.yml index a80fb35ab2c..6363505b941 100644 --- a/.github/workflows/lockfile-stats.lock.yml +++ b/.github/workflows/lockfile-stats.lock.yml @@ -1205,6 +1205,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1218,6 +1391,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/mattpocock-skills-reviewer.lock.yml b/.github/workflows/mattpocock-skills-reviewer.lock.yml index 5e6f15a4e0b..7aa848ebc7b 100644 --- a/.github/workflows/mattpocock-skills-reviewer.lock.yml +++ b/.github/workflows/mattpocock-skills-reviewer.lock.yml @@ -1232,6 +1232,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1245,6 +1418,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/mcp-inspector.lock.yml b/.github/workflows/mcp-inspector.lock.yml index 006ed014f23..d49cb136c25 100644 --- a/.github/workflows/mcp-inspector.lock.yml +++ b/.github/workflows/mcp-inspector.lock.yml @@ -1675,6 +1675,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1688,6 +1861,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/mergefest.lock.yml b/.github/workflows/mergefest.lock.yml index e7f821d50c4..0c5e2628e36 100644 --- a/.github/workflows/mergefest.lock.yml +++ b/.github/workflows/mergefest.lock.yml @@ -1193,6 +1193,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1206,6 +1379,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/metrics-collector.lock.yml b/.github/workflows/metrics-collector.lock.yml index 9f0f3f45fc2..e893dd6b91d 100644 --- a/.github/workflows/metrics-collector.lock.yml +++ b/.github/workflows/metrics-collector.lock.yml @@ -1235,6 +1235,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1248,6 +1421,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/necromancer.lock.yml b/.github/workflows/necromancer.lock.yml index 9eb5dbe35b7..0200e3e383a 100644 --- a/.github/workflows/necromancer.lock.yml +++ b/.github/workflows/necromancer.lock.yml @@ -1211,6 +1211,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1224,6 +1397,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/notion-issue-summary.lock.yml b/.github/workflows/notion-issue-summary.lock.yml index fa04057358c..a9cf6f9dcab 100644 --- a/.github/workflows/notion-issue-summary.lock.yml +++ b/.github/workflows/notion-issue-summary.lock.yml @@ -1107,6 +1107,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1120,6 +1293,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/objective-impact-report.lock.yml b/.github/workflows/objective-impact-report.lock.yml index fedf605f479..9bb8b3f31af 100644 --- a/.github/workflows/objective-impact-report.lock.yml +++ b/.github/workflows/objective-impact-report.lock.yml @@ -1112,6 +1112,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1125,6 +1298,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/org-health-report.lock.yml b/.github/workflows/org-health-report.lock.yml index 212e79da070..825be0435f6 100644 --- a/.github/workflows/org-health-report.lock.yml +++ b/.github/workflows/org-health-report.lock.yml @@ -1208,6 +1208,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1221,6 +1394,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/outcome-collector.lock.yml b/.github/workflows/outcome-collector.lock.yml index 7ef51e37fe6..52c6d9c2225 100644 --- a/.github/workflows/outcome-collector.lock.yml +++ b/.github/workflows/outcome-collector.lock.yml @@ -1155,6 +1155,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1168,6 +1341,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/pdf-summary.lock.yml b/.github/workflows/pdf-summary.lock.yml index c9fdd4cfefa..e00a929d180 100644 --- a/.github/workflows/pdf-summary.lock.yml +++ b/.github/workflows/pdf-summary.lock.yml @@ -1267,6 +1267,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1280,6 +1453,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/plan.lock.yml b/.github/workflows/plan.lock.yml index 38c58acef49..f26c7b897c9 100644 --- a/.github/workflows/plan.lock.yml +++ b/.github/workflows/plan.lock.yml @@ -1196,6 +1196,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1209,6 +1382,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/poem-bot.lock.yml b/.github/workflows/poem-bot.lock.yml index e4556d5d99b..0806bf67763 100644 --- a/.github/workflows/poem-bot.lock.yml +++ b/.github/workflows/poem-bot.lock.yml @@ -1478,6 +1478,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1491,6 +1664,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/portfolio-analyst.lock.yml b/.github/workflows/portfolio-analyst.lock.yml index 89f19cdbef6..8269113af8d 100644 --- a/.github/workflows/portfolio-analyst.lock.yml +++ b/.github/workflows/portfolio-analyst.lock.yml @@ -1334,6 +1334,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1347,6 +1520,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/pr-code-quality-reviewer.lock.yml b/.github/workflows/pr-code-quality-reviewer.lock.yml index 69d67e6b378..9e70da71fdf 100644 --- a/.github/workflows/pr-code-quality-reviewer.lock.yml +++ b/.github/workflows/pr-code-quality-reviewer.lock.yml @@ -1192,6 +1192,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1205,6 +1378,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/pr-description-caveman.lock.yml b/.github/workflows/pr-description-caveman.lock.yml index cbe239fa4cb..08df8ff6779 100644 --- a/.github/workflows/pr-description-caveman.lock.yml +++ b/.github/workflows/pr-description-caveman.lock.yml @@ -1114,6 +1114,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1127,6 +1300,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/pr-nitpick-reviewer.lock.yml b/.github/workflows/pr-nitpick-reviewer.lock.yml index 78ef7a054cc..7406d65c71e 100644 --- a/.github/workflows/pr-nitpick-reviewer.lock.yml +++ b/.github/workflows/pr-nitpick-reviewer.lock.yml @@ -1236,6 +1236,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1249,6 +1422,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/pr-sous-chef.lock.yml b/.github/workflows/pr-sous-chef.lock.yml index 58d8a2720f8..a55004f0818 100644 --- a/.github/workflows/pr-sous-chef.lock.yml +++ b/.github/workflows/pr-sous-chef.lock.yml @@ -1234,6 +1234,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1247,6 +1420,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/pr-triage-agent.lock.yml b/.github/workflows/pr-triage-agent.lock.yml index d944cad43a0..e544e172448 100644 --- a/.github/workflows/pr-triage-agent.lock.yml +++ b/.github/workflows/pr-triage-agent.lock.yml @@ -1260,6 +1260,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1273,6 +1446,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/prompt-clustering-analysis.lock.yml b/.github/workflows/prompt-clustering-analysis.lock.yml index 413897a5393..31c0ec916c1 100644 --- a/.github/workflows/prompt-clustering-analysis.lock.yml +++ b/.github/workflows/prompt-clustering-analysis.lock.yml @@ -1376,6 +1376,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1389,6 +1562,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/python-data-charts.lock.yml b/.github/workflows/python-data-charts.lock.yml index 6cb9c607bea..eefe8207f91 100644 --- a/.github/workflows/python-data-charts.lock.yml +++ b/.github/workflows/python-data-charts.lock.yml @@ -1291,6 +1291,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1304,6 +1477,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/q.lock.yml b/.github/workflows/q.lock.yml index d38c0309773..8a8fd934fab 100644 --- a/.github/workflows/q.lock.yml +++ b/.github/workflows/q.lock.yml @@ -1338,6 +1338,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1351,6 +1524,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/refactoring-cadence.lock.yml b/.github/workflows/refactoring-cadence.lock.yml index 21f8d1a25b1..45218804761 100644 --- a/.github/workflows/refactoring-cadence.lock.yml +++ b/.github/workflows/refactoring-cadence.lock.yml @@ -1148,6 +1148,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1161,6 +1334,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/refiner.lock.yml b/.github/workflows/refiner.lock.yml index db88ae41d44..6ac397606a3 100644 --- a/.github/workflows/refiner.lock.yml +++ b/.github/workflows/refiner.lock.yml @@ -1236,6 +1236,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1249,6 +1422,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/release.lock.yml b/.github/workflows/release.lock.yml index 3c665f0cd7d..a6c850dab74 100644 --- a/.github/workflows/release.lock.yml +++ b/.github/workflows/release.lock.yml @@ -1150,6 +1150,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1163,6 +1336,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/repo-audit-analyzer.lock.yml b/.github/workflows/repo-audit-analyzer.lock.yml index 93517caf218..fa8a92fc713 100644 --- a/.github/workflows/repo-audit-analyzer.lock.yml +++ b/.github/workflows/repo-audit-analyzer.lock.yml @@ -1146,6 +1146,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1159,6 +1332,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/repo-tree-map.lock.yml b/.github/workflows/repo-tree-map.lock.yml index 1ad6dcd2128..22a9e6dffed 100644 --- a/.github/workflows/repo-tree-map.lock.yml +++ b/.github/workflows/repo-tree-map.lock.yml @@ -1095,6 +1095,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1108,6 +1281,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/repository-quality-improver.lock.yml b/.github/workflows/repository-quality-improver.lock.yml index 1c440fc152c..a0c2a1e837c 100644 --- a/.github/workflows/repository-quality-improver.lock.yml +++ b/.github/workflows/repository-quality-improver.lock.yml @@ -1149,6 +1149,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1162,6 +1335,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/research.lock.yml b/.github/workflows/research.lock.yml index c5f55ae8236..3605dcfa970 100644 --- a/.github/workflows/research.lock.yml +++ b/.github/workflows/research.lock.yml @@ -1125,6 +1125,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1138,6 +1311,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/ruflo-backed-task.lock.yml b/.github/workflows/ruflo-backed-task.lock.yml index 236dc513172..efb6a3c8e67 100644 --- a/.github/workflows/ruflo-backed-task.lock.yml +++ b/.github/workflows/ruflo-backed-task.lock.yml @@ -1304,6 +1304,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1317,6 +1490,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/safe-output-health.lock.yml b/.github/workflows/safe-output-health.lock.yml index 9ff10ef10cf..ebd77df9bb6 100644 --- a/.github/workflows/safe-output-health.lock.yml +++ b/.github/workflows/safe-output-health.lock.yml @@ -1321,6 +1321,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1334,6 +1507,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/schema-consistency-checker.lock.yml b/.github/workflows/schema-consistency-checker.lock.yml index e0ca2551afd..96c44f4ba37 100644 --- a/.github/workflows/schema-consistency-checker.lock.yml +++ b/.github/workflows/schema-consistency-checker.lock.yml @@ -1122,6 +1122,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1135,6 +1308,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/schema-feature-coverage.lock.yml b/.github/workflows/schema-feature-coverage.lock.yml index 967f4ea0099..0ec75057950 100644 --- a/.github/workflows/schema-feature-coverage.lock.yml +++ b/.github/workflows/schema-feature-coverage.lock.yml @@ -1166,6 +1166,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1179,6 +1352,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/scout.lock.yml b/.github/workflows/scout.lock.yml index f7efe31856f..5a855c3e1ca 100644 --- a/.github/workflows/scout.lock.yml +++ b/.github/workflows/scout.lock.yml @@ -1403,6 +1403,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1416,6 +1589,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/security-compliance.lock.yml b/.github/workflows/security-compliance.lock.yml index 0e99287d652..2d31b0ab548 100644 --- a/.github/workflows/security-compliance.lock.yml +++ b/.github/workflows/security-compliance.lock.yml @@ -1154,6 +1154,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1167,6 +1340,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/security-review.lock.yml b/.github/workflows/security-review.lock.yml index 663cedfa2ca..603a1ebe61e 100644 --- a/.github/workflows/security-review.lock.yml +++ b/.github/workflows/security-review.lock.yml @@ -1280,6 +1280,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1293,6 +1466,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/semantic-function-refactor.lock.yml b/.github/workflows/semantic-function-refactor.lock.yml index d4e39e40c42..61d7cb8f865 100644 --- a/.github/workflows/semantic-function-refactor.lock.yml +++ b/.github/workflows/semantic-function-refactor.lock.yml @@ -1240,6 +1240,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1253,6 +1426,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/sergo.lock.yml b/.github/workflows/sergo.lock.yml index ef8937fbf0f..9bb7dad8ee6 100644 --- a/.github/workflows/sergo.lock.yml +++ b/.github/workflows/sergo.lock.yml @@ -1290,6 +1290,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1303,6 +1476,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/skillet.lock.yml b/.github/workflows/skillet.lock.yml index 83d06792aed..c9de8b0d749 100644 --- a/.github/workflows/skillet.lock.yml +++ b/.github/workflows/skillet.lock.yml @@ -1206,6 +1206,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1219,6 +1392,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/slide-deck-maintainer.lock.yml b/.github/workflows/slide-deck-maintainer.lock.yml index 57959547d07..15f58709dd9 100644 --- a/.github/workflows/slide-deck-maintainer.lock.yml +++ b/.github/workflows/slide-deck-maintainer.lock.yml @@ -1246,6 +1246,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1259,6 +1432,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-agent-all-merged.lock.yml b/.github/workflows/smoke-agent-all-merged.lock.yml index 7ebbafe0aa5..4b1a20c74b3 100644 --- a/.github/workflows/smoke-agent-all-merged.lock.yml +++ b/.github/workflows/smoke-agent-all-merged.lock.yml @@ -1206,6 +1206,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1219,6 +1392,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-agent-all-none.lock.yml b/.github/workflows/smoke-agent-all-none.lock.yml index 81b90fa8d8c..c605bbfae97 100644 --- a/.github/workflows/smoke-agent-all-none.lock.yml +++ b/.github/workflows/smoke-agent-all-none.lock.yml @@ -1206,6 +1206,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1219,6 +1392,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-agent-public-approved.lock.yml b/.github/workflows/smoke-agent-public-approved.lock.yml index 06f916bc2ae..4c16921a511 100644 --- a/.github/workflows/smoke-agent-public-approved.lock.yml +++ b/.github/workflows/smoke-agent-public-approved.lock.yml @@ -1237,6 +1237,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1250,6 +1423,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-agent-public-none.lock.yml b/.github/workflows/smoke-agent-public-none.lock.yml index 1da9e6c2d9c..378b2e56350 100644 --- a/.github/workflows/smoke-agent-public-none.lock.yml +++ b/.github/workflows/smoke-agent-public-none.lock.yml @@ -1206,6 +1206,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1219,6 +1392,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-agent-scoped-approved.lock.yml b/.github/workflows/smoke-agent-scoped-approved.lock.yml index c250b1e2d51..59b024cc41c 100644 --- a/.github/workflows/smoke-agent-scoped-approved.lock.yml +++ b/.github/workflows/smoke-agent-scoped-approved.lock.yml @@ -1213,6 +1213,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1226,6 +1399,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-antigravity.lock.yml b/.github/workflows/smoke-antigravity.lock.yml index 1702309aebf..56ae9323cac 100644 --- a/.github/workflows/smoke-antigravity.lock.yml +++ b/.github/workflows/smoke-antigravity.lock.yml @@ -1274,6 +1274,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1287,6 +1460,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-call-workflow.lock.yml b/.github/workflows/smoke-call-workflow.lock.yml index b0a3c3493eb..d7b7e25681d 100644 --- a/.github/workflows/smoke-call-workflow.lock.yml +++ b/.github/workflows/smoke-call-workflow.lock.yml @@ -1208,6 +1208,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1221,6 +1394,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-ci.lock.yml b/.github/workflows/smoke-ci.lock.yml index 87f4a5c0755..a7f24626dd8 100644 --- a/.github/workflows/smoke-ci.lock.yml +++ b/.github/workflows/smoke-ci.lock.yml @@ -1397,6 +1397,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1410,6 +1583,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-claude.lock.yml b/.github/workflows/smoke-claude.lock.yml index 0ff8e75ee25..c9c084e73ab 100644 --- a/.github/workflows/smoke-claude.lock.yml +++ b/.github/workflows/smoke-claude.lock.yml @@ -2041,6 +2041,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -2054,6 +2227,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-codex.lock.yml b/.github/workflows/smoke-codex.lock.yml index e182cc33ccb..bae7f03b419 100644 --- a/.github/workflows/smoke-codex.lock.yml +++ b/.github/workflows/smoke-codex.lock.yml @@ -1569,6 +1569,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1582,6 +1755,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-copilot-aoai-apikey.lock.yml b/.github/workflows/smoke-copilot-aoai-apikey.lock.yml index 6c75288de59..dd972b12420 100644 --- a/.github/workflows/smoke-copilot-aoai-apikey.lock.yml +++ b/.github/workflows/smoke-copilot-aoai-apikey.lock.yml @@ -2209,6 +2209,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -2222,6 +2395,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-copilot-aoai-entra.lock.yml b/.github/workflows/smoke-copilot-aoai-entra.lock.yml index e93b56cd0c9..8372c79cd7a 100644 --- a/.github/workflows/smoke-copilot-aoai-entra.lock.yml +++ b/.github/workflows/smoke-copilot-aoai-entra.lock.yml @@ -2213,6 +2213,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -2226,6 +2399,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-copilot-arm.lock.yml b/.github/workflows/smoke-copilot-arm.lock.yml index 692d6017350..25b0aa966a0 100644 --- a/.github/workflows/smoke-copilot-arm.lock.yml +++ b/.github/workflows/smoke-copilot-arm.lock.yml @@ -2067,6 +2067,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -2080,6 +2253,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-copilot-sdk.lock.yml b/.github/workflows/smoke-copilot-sdk.lock.yml index 7544d94b1f2..c0971db31c8 100644 --- a/.github/workflows/smoke-copilot-sdk.lock.yml +++ b/.github/workflows/smoke-copilot-sdk.lock.yml @@ -1143,6 +1143,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1156,6 +1329,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-copilot.lock.yml b/.github/workflows/smoke-copilot.lock.yml index cd5100ff72f..9dcff49fc9e 100644 --- a/.github/workflows/smoke-copilot.lock.yml +++ b/.github/workflows/smoke-copilot.lock.yml @@ -2211,6 +2211,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -2224,6 +2397,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-create-cross-repo-pr.lock.yml b/.github/workflows/smoke-create-cross-repo-pr.lock.yml index dfe41dd8e02..727dd97492e 100644 --- a/.github/workflows/smoke-create-cross-repo-pr.lock.yml +++ b/.github/workflows/smoke-create-cross-repo-pr.lock.yml @@ -1273,6 +1273,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1286,6 +1459,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-crush.lock.yml b/.github/workflows/smoke-crush.lock.yml index 78fc5d38872..967a7df7cde 100644 --- a/.github/workflows/smoke-crush.lock.yml +++ b/.github/workflows/smoke-crush.lock.yml @@ -1172,6 +1172,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1185,6 +1358,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-gemini.lock.yml b/.github/workflows/smoke-gemini.lock.yml index 112fee7bcac..894b315e2fa 100644 --- a/.github/workflows/smoke-gemini.lock.yml +++ b/.github/workflows/smoke-gemini.lock.yml @@ -1277,6 +1277,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1290,6 +1463,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-multi-pr.lock.yml b/.github/workflows/smoke-multi-pr.lock.yml index 24c0db8de53..dfadf3f3f9d 100644 --- a/.github/workflows/smoke-multi-pr.lock.yml +++ b/.github/workflows/smoke-multi-pr.lock.yml @@ -1218,6 +1218,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1231,6 +1404,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-opencode.lock.yml b/.github/workflows/smoke-opencode.lock.yml index ba4a22f7eed..f9bfd96e524 100644 --- a/.github/workflows/smoke-opencode.lock.yml +++ b/.github/workflows/smoke-opencode.lock.yml @@ -1177,6 +1177,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1190,6 +1363,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-otel-backends.lock.yml b/.github/workflows/smoke-otel-backends.lock.yml index 98c378e3ada..fd19c4e655c 100644 --- a/.github/workflows/smoke-otel-backends.lock.yml +++ b/.github/workflows/smoke-otel-backends.lock.yml @@ -1316,6 +1316,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1329,6 +1502,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-pi.lock.yml b/.github/workflows/smoke-pi.lock.yml index 5466f67566b..e0ef434a2d7 100644 --- a/.github/workflows/smoke-pi.lock.yml +++ b/.github/workflows/smoke-pi.lock.yml @@ -1230,6 +1230,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1243,6 +1416,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-project.lock.yml b/.github/workflows/smoke-project.lock.yml index 20a1ac7313f..09fc94dae47 100644 --- a/.github/workflows/smoke-project.lock.yml +++ b/.github/workflows/smoke-project.lock.yml @@ -1400,6 +1400,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1413,6 +1586,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-service-ports.lock.yml b/.github/workflows/smoke-service-ports.lock.yml index 73747200ffb..0dee877f7ce 100644 --- a/.github/workflows/smoke-service-ports.lock.yml +++ b/.github/workflows/smoke-service-ports.lock.yml @@ -1144,6 +1144,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1157,6 +1330,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-temporary-id.lock.yml b/.github/workflows/smoke-temporary-id.lock.yml index dc78dc81b53..75c9d569d67 100644 --- a/.github/workflows/smoke-temporary-id.lock.yml +++ b/.github/workflows/smoke-temporary-id.lock.yml @@ -1245,6 +1245,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1258,6 +1431,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-test-tools.lock.yml b/.github/workflows/smoke-test-tools.lock.yml index 70cc0cd9109..07e7e765cd3 100644 --- a/.github/workflows/smoke-test-tools.lock.yml +++ b/.github/workflows/smoke-test-tools.lock.yml @@ -1176,6 +1176,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1189,6 +1362,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-update-cross-repo-pr.lock.yml b/.github/workflows/smoke-update-cross-repo-pr.lock.yml index 1a658843c70..3f1817a34ec 100644 --- a/.github/workflows/smoke-update-cross-repo-pr.lock.yml +++ b/.github/workflows/smoke-update-cross-repo-pr.lock.yml @@ -1304,6 +1304,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1317,6 +1490,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-workflow-call-with-inputs.lock.yml b/.github/workflows/smoke-workflow-call-with-inputs.lock.yml index 211a5680f22..a2b904e2877 100644 --- a/.github/workflows/smoke-workflow-call-with-inputs.lock.yml +++ b/.github/workflows/smoke-workflow-call-with-inputs.lock.yml @@ -1201,6 +1201,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1214,6 +1387,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/smoke-workflow-call.lock.yml b/.github/workflows/smoke-workflow-call.lock.yml index 75bcebac4db..39e3ac236c9 100644 --- a/.github/workflows/smoke-workflow-call.lock.yml +++ b/.github/workflows/smoke-workflow-call.lock.yml @@ -1190,6 +1190,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1203,6 +1376,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/spec-enforcer.lock.yml b/.github/workflows/spec-enforcer.lock.yml index 5acbfb1f9af..b5ac61e92d3 100644 --- a/.github/workflows/spec-enforcer.lock.yml +++ b/.github/workflows/spec-enforcer.lock.yml @@ -1136,6 +1136,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1149,6 +1322,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/spec-extractor.lock.yml b/.github/workflows/spec-extractor.lock.yml index a5407f668fb..3f1b882e332 100644 --- a/.github/workflows/spec-extractor.lock.yml +++ b/.github/workflows/spec-extractor.lock.yml @@ -1229,6 +1229,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1242,6 +1415,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/spec-librarian.lock.yml b/.github/workflows/spec-librarian.lock.yml index 135b098a1c1..79bd78b0688 100644 --- a/.github/workflows/spec-librarian.lock.yml +++ b/.github/workflows/spec-librarian.lock.yml @@ -1190,6 +1190,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1203,6 +1376,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/stale-pr-cleanup.lock.yml b/.github/workflows/stale-pr-cleanup.lock.yml index 51b9d5dd28e..0a03f34b1d9 100644 --- a/.github/workflows/stale-pr-cleanup.lock.yml +++ b/.github/workflows/stale-pr-cleanup.lock.yml @@ -1137,6 +1137,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1150,6 +1323,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/stale-repo-identifier.lock.yml b/.github/workflows/stale-repo-identifier.lock.yml index 87b313acd33..97c062775ac 100644 --- a/.github/workflows/stale-repo-identifier.lock.yml +++ b/.github/workflows/stale-repo-identifier.lock.yml @@ -1333,6 +1333,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1346,6 +1519,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/static-analysis-report.lock.yml b/.github/workflows/static-analysis-report.lock.yml index e5b554db119..d6d792d8037 100644 --- a/.github/workflows/static-analysis-report.lock.yml +++ b/.github/workflows/static-analysis-report.lock.yml @@ -1346,6 +1346,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1359,6 +1532,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/step-name-alignment.lock.yml b/.github/workflows/step-name-alignment.lock.yml index 62d62afa8b7..afb6709f0f0 100644 --- a/.github/workflows/step-name-alignment.lock.yml +++ b/.github/workflows/step-name-alignment.lock.yml @@ -1232,6 +1232,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1245,6 +1418,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/sub-issue-closer.lock.yml b/.github/workflows/sub-issue-closer.lock.yml index d03ebe642da..065e2f6177e 100644 --- a/.github/workflows/sub-issue-closer.lock.yml +++ b/.github/workflows/sub-issue-closer.lock.yml @@ -1138,6 +1138,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1151,6 +1324,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/super-linter.lock.yml b/.github/workflows/super-linter.lock.yml index 41338fb51d9..955fa37cb60 100644 --- a/.github/workflows/super-linter.lock.yml +++ b/.github/workflows/super-linter.lock.yml @@ -1166,6 +1166,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1179,6 +1352,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/technical-doc-writer.lock.yml b/.github/workflows/technical-doc-writer.lock.yml index 6b77301b2fb..1c419a5ef74 100644 --- a/.github/workflows/technical-doc-writer.lock.yml +++ b/.github/workflows/technical-doc-writer.lock.yml @@ -1256,6 +1256,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1269,6 +1442,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/terminal-stylist.lock.yml b/.github/workflows/terminal-stylist.lock.yml index d0b7f48510f..50acac163cf 100644 --- a/.github/workflows/terminal-stylist.lock.yml +++ b/.github/workflows/terminal-stylist.lock.yml @@ -1128,6 +1128,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1141,6 +1314,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/test-create-pr-error-handling.lock.yml b/.github/workflows/test-create-pr-error-handling.lock.yml index 76192ac9eeb..385249c86b2 100644 --- a/.github/workflows/test-create-pr-error-handling.lock.yml +++ b/.github/workflows/test-create-pr-error-handling.lock.yml @@ -1213,6 +1213,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1226,6 +1399,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/test-dispatcher.lock.yml b/.github/workflows/test-dispatcher.lock.yml index 4eaebd4ef56..bb885ca3248 100644 --- a/.github/workflows/test-dispatcher.lock.yml +++ b/.github/workflows/test-dispatcher.lock.yml @@ -1093,6 +1093,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1106,6 +1279,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/test-project-url-default.lock.yml b/.github/workflows/test-project-url-default.lock.yml index 16e9eaedb3a..d6e0ab293d1 100644 --- a/.github/workflows/test-project-url-default.lock.yml +++ b/.github/workflows/test-project-url-default.lock.yml @@ -1139,6 +1139,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1152,6 +1325,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/test-quality-sentinel.lock.yml b/.github/workflows/test-quality-sentinel.lock.yml index 06dbc310877..8f72df0cad8 100644 --- a/.github/workflows/test-quality-sentinel.lock.yml +++ b/.github/workflows/test-quality-sentinel.lock.yml @@ -1206,6 +1206,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1219,6 +1392,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/test-workflow.lock.yml b/.github/workflows/test-workflow.lock.yml index 6c4ce672327..914946bb219 100644 --- a/.github/workflows/test-workflow.lock.yml +++ b/.github/workflows/test-workflow.lock.yml @@ -1016,6 +1016,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1029,6 +1202,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/tidy.lock.yml b/.github/workflows/tidy.lock.yml index 0cc8ff8954c..311fccb3774 100644 --- a/.github/workflows/tidy.lock.yml +++ b/.github/workflows/tidy.lock.yml @@ -1235,6 +1235,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1248,6 +1421,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/typist.lock.yml b/.github/workflows/typist.lock.yml index 8856dafd6f7..268116d72b5 100644 --- a/.github/workflows/typist.lock.yml +++ b/.github/workflows/typist.lock.yml @@ -1253,6 +1253,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1266,6 +1439,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/ubuntu-image-analyzer.lock.yml b/.github/workflows/ubuntu-image-analyzer.lock.yml index f91ee768476..704a6d36e78 100644 --- a/.github/workflows/ubuntu-image-analyzer.lock.yml +++ b/.github/workflows/ubuntu-image-analyzer.lock.yml @@ -1149,6 +1149,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1162,6 +1335,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/uk-ai-operational-resilience.lock.yml b/.github/workflows/uk-ai-operational-resilience.lock.yml index 0e39551ec82..2033fb26443 100644 --- a/.github/workflows/uk-ai-operational-resilience.lock.yml +++ b/.github/workflows/uk-ai-operational-resilience.lock.yml @@ -1129,6 +1129,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1142,6 +1315,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/unbloat-docs.lock.yml b/.github/workflows/unbloat-docs.lock.yml index eb1175ce5a2..2195eb2504a 100644 --- a/.github/workflows/unbloat-docs.lock.yml +++ b/.github/workflows/unbloat-docs.lock.yml @@ -1227,6 +1227,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1240,6 +1413,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/update-astro.lock.yml b/.github/workflows/update-astro.lock.yml index c3628be1b62..d1e1b5b9228 100644 --- a/.github/workflows/update-astro.lock.yml +++ b/.github/workflows/update-astro.lock.yml @@ -1174,6 +1174,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1187,6 +1360,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/video-analyzer.lock.yml b/.github/workflows/video-analyzer.lock.yml index 8279ee63207..3a55047212d 100644 --- a/.github/workflows/video-analyzer.lock.yml +++ b/.github/workflows/video-analyzer.lock.yml @@ -1116,6 +1116,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1129,6 +1302,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/visual-regression-checker.lock.yml b/.github/workflows/visual-regression-checker.lock.yml index 6742cec4e17..1a019ce7d5a 100644 --- a/.github/workflows/visual-regression-checker.lock.yml +++ b/.github/workflows/visual-regression-checker.lock.yml @@ -1186,6 +1186,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1199,6 +1372,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/weekly-blog-post-writer.lock.yml b/.github/workflows/weekly-blog-post-writer.lock.yml index 679409a9add..8d6abc38351 100644 --- a/.github/workflows/weekly-blog-post-writer.lock.yml +++ b/.github/workflows/weekly-blog-post-writer.lock.yml @@ -1314,6 +1314,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1327,6 +1500,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/weekly-editors-health-check.lock.yml b/.github/workflows/weekly-editors-health-check.lock.yml index 51b43c17a8a..4866728801f 100644 --- a/.github/workflows/weekly-editors-health-check.lock.yml +++ b/.github/workflows/weekly-editors-health-check.lock.yml @@ -1183,6 +1183,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1196,6 +1369,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/weekly-issue-summary.lock.yml b/.github/workflows/weekly-issue-summary.lock.yml index 19e98e7f00b..ab5b3b7ed24 100644 --- a/.github/workflows/weekly-issue-summary.lock.yml +++ b/.github/workflows/weekly-issue-summary.lock.yml @@ -1171,6 +1171,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1184,6 +1357,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/weekly-safe-outputs-spec-review.lock.yml b/.github/workflows/weekly-safe-outputs-spec-review.lock.yml index b6d3bd24937..ba1de1cd350 100644 --- a/.github/workflows/weekly-safe-outputs-spec-review.lock.yml +++ b/.github/workflows/weekly-safe-outputs-spec-review.lock.yml @@ -1106,6 +1106,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1119,6 +1292,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/workflow-generator.lock.yml b/.github/workflows/workflow-generator.lock.yml index fc9bf71a0a8..544ff86c966 100644 --- a/.github/workflows/workflow-generator.lock.yml +++ b/.github/workflows/workflow-generator.lock.yml @@ -1179,6 +1179,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1192,6 +1365,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/workflow-health-manager.lock.yml b/.github/workflows/workflow-health-manager.lock.yml index b0dc91e73b7..c6d84164713 100644 --- a/.github/workflows/workflow-health-manager.lock.yml +++ b/.github/workflows/workflow-health-manager.lock.yml @@ -1225,6 +1225,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1238,6 +1411,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/workflow-normalizer.lock.yml b/.github/workflows/workflow-normalizer.lock.yml index c7bbba7d0ba..9c862caefd1 100644 --- a/.github/workflows/workflow-normalizer.lock.yml +++ b/.github/workflows/workflow-normalizer.lock.yml @@ -1189,6 +1189,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1202,6 +1375,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion diff --git a/.github/workflows/workflow-skill-extractor.lock.yml b/.github/workflows/workflow-skill-extractor.lock.yml index c0759a5d27e..a0c97942eca 100644 --- a/.github/workflows/workflow-skill-extractor.lock.yml +++ b/.github/workflows/workflow-skill-extractor.lock.yml @@ -1160,6 +1160,179 @@ jobs: [ -f /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl || true [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl + mkdir -p /tmp/gh-aw/usage/activity + python3 - <<'PY' + import glob + import json + import os + + # NOTE: this aggregation script intentionally stays inline in the generated + # workflow step so compiled workflows are self-contained and do not depend on + # extra repository files at runtime. + # usage-activity-summary/v1 structure: + # firewall: total/allowed/blocked request counters + # session: aggregate Copilot session event counters + # gateway: total/failed tool-call counters with per-server breakdown + summary = {'schema': 'usage-activity-summary/v1'} + SQUID_STATUS_INDEX = 6 + SQUID_DECISION_INDEX = 7 + + firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + firewall_paths = [ + '/tmp/gh-aw/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', + '/tmp/gh-aw/squid-logs-*/*.log', + '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', + ] + for pattern in firewall_paths: + for log_path in glob.glob(pattern): + try: + with open(log_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or line.startswith('#'): + continue + parts = line.split() + if len(parts) < 8: + continue + firewall['total_requests'] += 1 + # Squid access log columns (0-based): + # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + # 6=status 7=decision 8=url 9=user-agent + # Keep indices named for easier maintenance if format changes. + status = parts[SQUID_STATUS_INDEX] + decision = parts[SQUID_DECISION_INDEX] + allowed = False + try: + code = int(status) + allowed = code in (200, 206, 304) + except ValueError: + allowed = False + if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + allowed = True + if allowed: + firewall['allowed_requests'] += 1 + else: + firewall['blocked_requests'] += 1 + except OSError: + continue + if firewall['total_requests'] > 0: + summary['firewall'] = firewall + + session = { + 'total_events': 0, + 'session_starts': 0, + 'session_shutdowns': 0, + 'turns': 0, + 'assistant_messages': 0, + 'reasoning_events': 0, + 'tool_execution_starts': 0, + 'tool_execution_completes': 0, + 'failed_tool_executions': 0, + } + session_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', + ] + for pattern in session_paths: + for events_path in glob.glob(pattern): + try: + with open(events_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event_type = str(entry.get('type', '')).strip().lower() + session['total_events'] += 1 + if event_type == 'session.start': + session['session_starts'] += 1 + elif event_type == 'session.shutdown': + session['session_shutdowns'] += 1 + elif event_type == 'user.message': + session['turns'] += 1 + elif event_type == 'assistant.message': + session['assistant_messages'] += 1 + # Copilot session logs use both reasoning and assistant.reasoning + # across CLI/runtime versions, so count both as reasoning events. + elif event_type in ('reasoning', 'assistant.reasoning'): + session['reasoning_events'] += 1 + elif event_type == 'tool.execution_start': + session['tool_execution_starts'] += 1 + elif event_type == 'tool.execution_complete': + session['tool_execution_completes'] += 1 + data = entry.get('data', {}) + success = True + if isinstance(data, dict): + success = bool(data.get('success', True)) + if not success: + session['failed_tool_executions'] += 1 + except OSError: + continue + if session['total_events'] > 0: + summary['session'] = session + + gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} + gateway_paths = [ + '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', + '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', + ] + for gateway_path in gateway_paths: + if not os.path.exists(gateway_path): + continue + try: + with open(gateway_path, encoding='utf-8', errors='ignore') as handle: + for raw in handle: + line = raw.strip() + if not line or not line.startswith('{'): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + event = str(entry.get('event', '')).strip().lower() + if event not in ('tool_call', 'rpc_call', 'request'): + continue + gateway['total_calls'] += 1 + status = str(entry.get('status', '')).strip().lower() + level = str(entry.get('level', '')).strip().lower() + error_text = str(entry.get('error', '')).strip() + failed = status == 'error' or error_text != '' or level == 'error' + if failed: + gateway['failed_calls'] += 1 + # gateway.jsonl has server_name for modern logs and server_id in + # some compatibility/transition paths; keep fallback ordering explicit. + server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') + server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) + server_bucket['tool_call_count'] += 1 + if failed: + server_bucket['failed_calls'] += 1 + except OSError: + continue + if gateway['total_calls'] > 0: + summary['gateway'] = { + 'total_calls': gateway['total_calls'], + 'failed_calls': gateway['failed_calls'], + 'servers': [ + { + 'server_name': server_name, + 'tool_call_count': bucket['tool_call_count'], + 'failed_calls': bucket['failed_calls'], + } + for server_name, bucket in sorted(gateway['servers'].items()) + ], + } + + output_path = '/tmp/gh-aw/usage/activity/summary.json' + with open(output_path, 'w', encoding='utf-8') as handle: + json.dump(summary, handle, sort_keys=True) + print(output_path) + PY find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() @@ -1173,6 +1346,7 @@ jobs: /tmp/gh-aw/usage/detection_usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl + /tmp/gh-aw/usage/activity/summary.json if-no-files-found: ignore - name: Restore daily AIC usage cache id: restore-daily-aic-cache-conclusion From d69fe2b2018d6a3ef06e5a4f00485efacdbc1266 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 20 Jun 2026 20:54:22 +0000 Subject: [PATCH 12/17] Fix usage activity backfill and include rate limit logs Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/cli/logs_run_processor.go | 2 +- pkg/cli/logs_usage_activity.go | 20 +++- pkg/cli/logs_usage_activity_test.go | 136 +++++++++++++++++++++++----- pkg/workflow/notify_comment.go | 25 +++-- pkg/workflow/notify_comment_test.go | 9 ++ 5 files changed, 156 insertions(+), 36 deletions(-) diff --git a/pkg/cli/logs_run_processor.go b/pkg/cli/logs_run_processor.go index c1098a9380a..06d96cc2168 100644 --- a/pkg/cli/logs_run_processor.go +++ b/pkg/cli/logs_run_processor.go @@ -339,7 +339,7 @@ func downloadRunArtifactsConcurrent(ctx context.Context, runs []WorkflowRun, out result.GitHubRateLimitUsage = rateLimitUsage // Fill missing activity summaries from usage artifact precomputes. // This call is unconditional but only backfills fields that are still empty. - applyUsageActivitySummaryToResult(usageActivitySummary, &result) + applyUsageActivitySummaryToResult(usageActivitySummary, &result, !hasFirewallArtifact) // Count safe output items created in GitHub (from manifest artifact) result.Run.SafeItemsCount = len(extractCreatedItemsFromManifest(runOutputDir)) diff --git a/pkg/cli/logs_usage_activity.go b/pkg/cli/logs_usage_activity.go index 0604bb29c45..a1a2fb7f15b 100644 --- a/pkg/cli/logs_usage_activity.go +++ b/pkg/cli/logs_usage_activity.go @@ -7,6 +7,8 @@ import ( "path/filepath" ) +const usageActivitySummarySchema = "usage-activity-summary/v1" + type usageActivitySummary struct { Schema string `json:"schema,omitempty"` Firewall *usageActivityFirewall `json:"firewall,omitempty"` @@ -49,6 +51,7 @@ func loadUsageActivitySummary(runDir string) (*usageActivitySummary, error) { filepath.Join(runDir, "usage", "activity", "summary.json"), filepath.Join(runDir, "activity", "summary.json"), } + var lastErr error for _, candidate := range candidates { cleanPath := filepath.Clean(candidate) raw, err := os.ReadFile(cleanPath) @@ -60,21 +63,26 @@ func loadUsageActivitySummary(runDir string) (*usageActivitySummary, error) { } var summary usageActivitySummary if err := json.Unmarshal(raw, &summary); err != nil { - return nil, fmt.Errorf("parse usage activity summary %s: %w", cleanPath, err) + lastErr = fmt.Errorf("parse usage activity summary %s: %w", cleanPath, err) + continue + } + if summary.Schema != usageActivitySummarySchema { + lastErr = fmt.Errorf("unsupported usage activity summary schema %q in %s (expected %q)", summary.Schema, cleanPath, usageActivitySummarySchema) + continue } return &summary, nil } - return nil, nil + return nil, lastErr } -func applyUsageActivitySummaryToResult(summary *usageActivitySummary, result *DownloadResult) { +func applyUsageActivitySummaryToResult(summary *usageActivitySummary, result *DownloadResult, allowTurnBackfill bool) { if summary == nil || result == nil { return } // Preserve previously parsed turn counts (from full session artifacts/events.jsonl) // and only backfill when they are missing. - if summary.Session != nil && result.Run.Turns == 0 && summary.Session.Turns > 0 { + if allowTurnBackfill && summary.Session != nil && result.Run.Turns == 0 && summary.Session.Turns > 0 { result.Run.Turns = summary.Session.Turns } @@ -102,7 +110,9 @@ func applyUsageActivitySummaryToResult(summary *usageActivitySummary, result *Do }) } result.MCPToolUsage = &MCPToolUsageData{ - Servers: servers, + Summary: []MCPToolSummary{}, + ToolCalls: []MCPToolCall{}, + Servers: servers, } } } diff --git a/pkg/cli/logs_usage_activity_test.go b/pkg/cli/logs_usage_activity_test.go index f5dc3bf0d94..4f5138343a3 100644 --- a/pkg/cli/logs_usage_activity_test.go +++ b/pkg/cli/logs_usage_activity_test.go @@ -1,3 +1,5 @@ +//go:build !integration + package cli import ( @@ -14,23 +16,23 @@ func TestLoadUsageActivitySummary(t *testing.T) { runDir := t.TempDir() summaryPath := filepath.Join(runDir, "usage", "activity", "summary.json") - require.NoError(t, os.MkdirAll(filepath.Dir(summaryPath), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Dir(summaryPath), 0o755), "should create usage activity directory") require.NoError(t, os.WriteFile(summaryPath, []byte(`{ - "schema":"usage-activity-summary/v1", + "schema":"`+usageActivitySummarySchema+`", "firewall":{"total_requests":10,"allowed_requests":8,"blocked_requests":2}, "session":{"turns":7}, "gateway":{"total_calls":5,"failed_calls":1} - }`), 0o644)) + }`), 0o644), "should write usage activity summary") summary, err := loadUsageActivitySummary(runDir) - require.NoError(t, err) - require.NotNil(t, summary) - require.NotNil(t, summary.Firewall) - assert.Equal(t, 10, summary.Firewall.TotalRequests) - require.NotNil(t, summary.Session) - assert.Equal(t, 7, summary.Session.Turns) - require.NotNil(t, summary.Gateway) - assert.Equal(t, 5, summary.Gateway.TotalCalls) + require.NoError(t, err, "loadUsageActivitySummary should parse the primary usage path") + require.NotNil(t, summary, "summary should not be nil") + require.NotNil(t, summary.Firewall, "firewall section should be present") + assert.Equal(t, 10, summary.Firewall.TotalRequests, "firewall total_requests should be parsed from JSON") + require.NotNil(t, summary.Session, "session section should be present") + assert.Equal(t, 7, summary.Session.Turns, "session turns should be parsed from JSON") + require.NotNil(t, summary.Gateway, "gateway section should be present") + assert.Equal(t, 5, summary.Gateway.TotalCalls, "gateway total_calls should be parsed from JSON") } func TestApplyUsageActivitySummaryToResult(t *testing.T) { @@ -54,15 +56,105 @@ func TestApplyUsageActivitySummaryToResult(t *testing.T) { }, } - applyUsageActivitySummaryToResult(summary, &result) - - assert.Equal(t, 4, result.Run.Turns) - require.NotNil(t, result.FirewallAnalysis) - assert.Equal(t, 12, result.FirewallAnalysis.TotalRequests) - assert.Equal(t, 3, result.FirewallAnalysis.BlockedRequests) - require.NotNil(t, result.MCPToolUsage) - require.Len(t, result.MCPToolUsage.Servers, 2) - assert.Equal(t, "github", result.MCPToolUsage.Servers[0].ServerName) - assert.Equal(t, 5, result.MCPToolUsage.Servers[0].ToolCallCount) - assert.Equal(t, 2, result.MCPToolUsage.Servers[0].ErrorCount) + applyUsageActivitySummaryToResult(summary, &result, true) + + assert.Equal(t, 4, result.Run.Turns, "turns should be backfilled when detailed session artifacts are absent") + require.NotNil(t, result.FirewallAnalysis, "firewall summary should be backfilled") + assert.Equal(t, 12, result.FirewallAnalysis.TotalRequests, "firewall total requests should be copied from the summary") + assert.Equal(t, 3, result.FirewallAnalysis.BlockedRequests, "firewall blocked requests should be copied from the summary") + require.NotNil(t, result.MCPToolUsage, "gateway summary should be backfilled") + assert.Empty(t, result.MCPToolUsage.Summary, "usage-summary backfill should preserve empty summary rows instead of null") + assert.Empty(t, result.MCPToolUsage.ToolCalls, "usage-summary backfill should preserve empty tool call rows instead of null") + require.Len(t, result.MCPToolUsage.Servers, 2, "gateway servers should be copied from the summary") + assert.Equal(t, "github", result.MCPToolUsage.Servers[0].ServerName, "server names should be preserved") + assert.Equal(t, 5, result.MCPToolUsage.Servers[0].ToolCallCount, "tool call counts should be preserved") + assert.Equal(t, 2, result.MCPToolUsage.Servers[0].ErrorCount, "failed call counts should map to server error counts") +} + +func TestLoadUsageActivitySummaryFallbackPath(t *testing.T) { + t.Parallel() + + runDir := t.TempDir() + fallbackPath := filepath.Join(runDir, "activity", "summary.json") + require.NoError(t, os.MkdirAll(filepath.Dir(fallbackPath), 0o755), "should create fallback activity directory") + require.NoError(t, os.WriteFile(fallbackPath, []byte(`{"schema":"`+usageActivitySummarySchema+`","session":{"turns":3}}`), 0o644), "should write fallback activity summary") + + summary, err := loadUsageActivitySummary(runDir) + require.NoError(t, err, "fallback activity summary should load without error") + require.NotNil(t, summary, "summary should be loaded from the fallback path") + require.NotNil(t, summary.Session, "session section should be present in the fallback summary") + assert.Equal(t, 3, summary.Session.Turns, "session turns should be loaded from the fallback path") +} + +func TestLoadUsageActivitySummaryNoFile(t *testing.T) { + t.Parallel() + + summary, err := loadUsageActivitySummary(t.TempDir()) + require.NoError(t, err, "missing activity summary should not be treated as an error") + assert.Nil(t, summary, "missing activity summary should return nil") +} + +func TestLoadUsageActivitySummaryMalformedPrimaryFallsBack(t *testing.T) { + t.Parallel() + + runDir := t.TempDir() + primaryPath := filepath.Join(runDir, "usage", "activity", "summary.json") + require.NoError(t, os.MkdirAll(filepath.Dir(primaryPath), 0o755), "should create primary activity directory") + require.NoError(t, os.WriteFile(primaryPath, []byte(`{not valid json`), 0o644), "should write malformed primary summary") + + fallbackPath := filepath.Join(runDir, "activity", "summary.json") + require.NoError(t, os.MkdirAll(filepath.Dir(fallbackPath), 0o755), "should create fallback activity directory") + require.NoError(t, os.WriteFile(fallbackPath, []byte(`{"schema":"`+usageActivitySummarySchema+`","session":{"turns":5}}`), 0o644), "should write valid fallback summary") + + summary, err := loadUsageActivitySummary(runDir) + require.NoError(t, err, "valid fallback summary should be used when the primary summary is malformed") + require.NotNil(t, summary, "fallback summary should be returned") + require.NotNil(t, summary.Session, "session section should be present after fallback") + assert.Equal(t, 5, summary.Session.Turns, "fallback session turns should be preserved") +} + +func TestLoadUsageActivitySummaryRejectsUnsupportedSchema(t *testing.T) { + t.Parallel() + + runDir := t.TempDir() + summaryPath := filepath.Join(runDir, "usage", "activity", "summary.json") + require.NoError(t, os.MkdirAll(filepath.Dir(summaryPath), 0o755), "should create usage activity directory") + require.NoError(t, os.WriteFile(summaryPath, []byte(`{"schema":"usage-activity-summary/v2"}`), 0o644), "should write unsupported schema summary") + + summary, err := loadUsageActivitySummary(runDir) + require.Error(t, err, "unsupported activity summary schema should return an error") + assert.Nil(t, summary, "unsupported schema should not be returned") + assert.Contains(t, err.Error(), "unsupported usage activity summary schema", "schema validation error should explain the mismatch") +} + +func TestApplyUsageActivitySummaryDoesNotOverwriteExistingData(t *testing.T) { + t.Parallel() + + existingFirewall := &FirewallAnalysis{TotalRequests: 100} + existingMCP := &MCPToolUsageData{ + Summary: []MCPToolSummary{}, + ToolCalls: []MCPToolCall{}, + } + result := DownloadResult{ + Run: WorkflowRun{Turns: 9}, + FirewallAnalysis: existingFirewall, + MCPToolUsage: existingMCP, + } + summary := &usageActivitySummary{ + Session: &usageActivitySession{Turns: 4}, + Firewall: &usageActivityFirewall{ + TotalRequests: 12, + AllowedRequests: 9, + BlockedRequests: 3, + }, + Gateway: &usageActivityGateway{ + Servers: []usageActivityGatewayServer{{ServerName: "github", ToolCallCount: 5, FailedCalls: 2}}, + }, + } + + applyUsageActivitySummaryToResult(summary, &result, false) + + assert.Equal(t, 9, result.Run.Turns, "existing turns must not be overwritten when detailed artifacts are available") + assert.Same(t, existingFirewall, result.FirewallAnalysis, "existing firewall analysis must not be replaced") + assert.Same(t, existingMCP, result.MCPToolUsage, "existing MCP tool usage must not be replaced") } diff --git a/pkg/workflow/notify_comment.go b/pkg/workflow/notify_comment.go index ac6e8b420bd..27df841eaba 100644 --- a/pkg/workflow/notify_comment.go +++ b/pkg/workflow/notify_comment.go @@ -689,12 +689,13 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " run: |\n", " mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection\n", " echo \"Usage artifact source file status:\"\n", - " for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do\n", + " for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do\n", " [ -f \"$file\" ] && echo \"FOUND: $file\" || echo \"MISSING: $file\"\n", " done\n", " [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true\n", " [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true\n", " [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true\n", + " [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true\n", " [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true\n", " [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true\n", " [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true\n", @@ -721,6 +722,10 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " SQUID_DECISION_INDEX = 7\n", "\n", " firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0}\n", + " def is_allowed_decision(decision: str) -> bool:\n", + " base = decision.split('/', 1)[0].strip().upper()\n", + " return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')\n", + "\n", " firewall_paths = [\n", " '/tmp/gh-aw/sandbox/firewall/logs/*.log',\n", " '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log',\n", @@ -751,7 +756,7 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " allowed = code in (200, 206, 304)\n", " except ValueError:\n", " allowed = False\n", - " if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')):\n", + " if not allowed and is_allowed_decision(decision):\n", " allowed = True\n", " if allowed:\n", " firewall['allowed_requests'] += 1\n", @@ -819,12 +824,15 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " summary['session'] = session\n", "\n", " gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}}\n", - " gateway_paths = [\n", - " '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl',\n", - " '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl',\n", - " '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl',\n", - " '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl',\n", - " ]\n", + " gateway_paths = []\n", + " for modern_path, legacy_path in [\n", + " ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'),\n", + " ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'),\n", + " ]:\n", + " if os.path.exists(modern_path):\n", + " gateway_paths.append(modern_path)\n", + " elif os.path.exists(legacy_path):\n", + " gateway_paths.append(legacy_path)\n", " for gateway_path in gateway_paths:\n", " if not os.path.exists(gateway_path):\n", " continue\n", @@ -887,6 +895,7 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " /tmp/gh-aw/usage/aw-info.jsonl\n", " /tmp/gh-aw/usage/agent_usage.jsonl\n", " /tmp/gh-aw/usage/detection_usage.jsonl\n", + " /tmp/gh-aw/usage/github_rate_limits.jsonl\n", " /tmp/gh-aw/usage/agent/token_usage.jsonl\n", " /tmp/gh-aw/usage/detection/token_usage.jsonl\n", " /tmp/gh-aw/usage/activity/summary.json\n", diff --git a/pkg/workflow/notify_comment_test.go b/pkg/workflow/notify_comment_test.go index 0a0cc9149db..7c99dcf9a59 100644 --- a/pkg/workflow/notify_comment_test.go +++ b/pkg/workflow/notify_comment_test.go @@ -1192,6 +1192,9 @@ func TestConclusionJobIncludesUsageArtifactSteps(t *testing.T) { if !strings.Contains(allSteps, "/tmp/gh-aw/usage/detection_usage.jsonl") { t.Errorf("Expected usage artifact to include detection_usage.jsonl path.\nGenerated steps:\n%s", allSteps) } + if !strings.Contains(allSteps, "/tmp/gh-aw/usage/github_rate_limits.jsonl") { + t.Errorf("Expected usage artifact to include GitHub API rate limit usage path.\nGenerated steps:\n%s", allSteps) + } if !strings.Contains(allSteps, "/tmp/gh-aw/usage/agent/token_usage.jsonl") { t.Errorf("Expected usage artifact to include agent token usage path.\nGenerated steps:\n%s", allSteps) } @@ -1216,6 +1219,12 @@ func TestConclusionJobIncludesUsageArtifactSteps(t *testing.T) { if !strings.Contains(allSteps, "python3 - <<'PY'") { t.Errorf("Expected usage artifact collection to generate activity summary aggregates.\nGenerated steps:\n%s", allSteps) } + if !strings.Contains(allSteps, "usage-activity-summary/v1") { + t.Errorf("Expected activity summary generator to emit the usage activity schema marker.\nGenerated steps:\n%s", allSteps) + } + if !strings.Contains(allSteps, "def is_allowed_decision(decision: str) -> bool:") { + t.Errorf("Expected activity summary generator to normalize Squid decision markers before counting allowed requests.\nGenerated steps:\n%s", allSteps) + } if !strings.Contains(allSteps, "/tmp/gh-aw/usage/activity/summary.json") { t.Errorf("Expected usage artifact to include activity summary path.\nGenerated steps:\n%s", allSteps) } From febe49c80681e4e883ed5e55ccef494ba22a0bea Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 20 Jun 2026 21:02:28 +0000 Subject: [PATCH 13/17] Apply remaining changes Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/ab-testing-advisor.lock.yml | 25 +++++++++++++------ .github/workflows/ace-editor.lock.yml | 25 +++++++++++++------ .../agent-performance-analyzer.lock.yml | 25 +++++++++++++------ .../workflows/agent-persona-explorer.lock.yml | 25 +++++++++++++------ .../workflows/agentic-token-audit.lock.yml | 25 +++++++++++++------ .../agentic-token-optimizer.lock.yml | 25 +++++++++++++------ .../agentic-token-trend-audit.lock.yml | 25 +++++++++++++------ .github/workflows/ai-moderator.lock.yml | 25 +++++++++++++------ .../workflows/api-consumption-report.lock.yml | 25 +++++++++++++------ .github/workflows/approach-validator.lock.yml | 25 +++++++++++++------ .github/workflows/archie.lock.yml | 25 +++++++++++++------ .../workflows/architecture-guardian.lock.yml | 25 +++++++++++++------ .github/workflows/artifacts-summary.lock.yml | 25 +++++++++++++------ .github/workflows/audit-workflows.lock.yml | 25 +++++++++++++------ .github/workflows/auto-triage-issues.lock.yml | 25 +++++++++++++------ .github/workflows/avenger.lock.yml | 25 +++++++++++++------ .../aw-failure-investigator.lock.yml | 25 +++++++++++++------ .github/workflows/blog-auditor.lock.yml | 25 +++++++++++++------ .github/workflows/bot-detection.lock.yml | 25 +++++++++++++------ .github/workflows/brave.lock.yml | 25 +++++++++++++------ .../breaking-change-checker.lock.yml | 25 +++++++++++++------ .github/workflows/changeset.lock.yml | 25 +++++++++++++------ .../workflows/chaos-pr-bundle-fuzzer.lock.yml | 25 +++++++++++++------ .github/workflows/ci-coach.lock.yml | 25 +++++++++++++------ .github/workflows/ci-doctor.lock.yml | 25 +++++++++++++------ .../claude-code-user-docs-review.lock.yml | 25 +++++++++++++------ .../cli-consistency-checker.lock.yml | 25 +++++++++++++------ .../workflows/cli-version-checker.lock.yml | 25 +++++++++++++------ .github/workflows/cloclo.lock.yml | 25 +++++++++++++------ .../workflows/code-scanning-fixer.lock.yml | 25 +++++++++++++------ .github/workflows/code-simplifier.lock.yml | 25 +++++++++++++------ .../codex-github-remote-mcp-test.lock.yml | 25 +++++++++++++------ .../commit-changes-analyzer.lock.yml | 25 +++++++++++++------ .../constraint-solving-potd.lock.yml | 25 +++++++++++++------ .github/workflows/contribution-check.lock.yml | 25 +++++++++++++------ .../workflows/copilot-agent-analysis.lock.yml | 25 +++++++++++++------ .../copilot-centralization-drilldown.lock.yml | 25 +++++++++++++------ .../copilot-centralization-optimizer.lock.yml | 25 +++++++++++++------ .../copilot-cli-deep-research.lock.yml | 25 +++++++++++++------ .github/workflows/copilot-opt.lock.yml | 25 +++++++++++++------ .../copilot-pr-merged-report.lock.yml | 25 +++++++++++++------ .../copilot-pr-nlp-analysis.lock.yml | 25 +++++++++++++------ .../copilot-pr-prompt-analysis.lock.yml | 25 +++++++++++++------ .../copilot-session-insights.lock.yml | 25 +++++++++++++------ .github/workflows/craft.lock.yml | 25 +++++++++++++------ ...aily-agent-of-the-day-blog-writer.lock.yml | 25 +++++++++++++------ .../daily-agentrx-trace-optimizer.lock.yml | 25 +++++++++++++------ .../daily-ambient-context-optimizer.lock.yml | 25 +++++++++++++------ .../daily-architecture-diagram.lock.yml | 25 +++++++++++++------ .../daily-assign-issue-to-user.lock.yml | 25 +++++++++++++------ ...strostylelite-markdown-spellcheck.lock.yml | 25 +++++++++++++------ ...daily-aw-cross-repo-compile-check.lock.yml | 25 +++++++++++++------ ...daily-awf-spec-compiler-surfacing.lock.yml | 25 +++++++++++++------ .../workflows/daily-byok-ollama-test.lock.yml | 25 +++++++++++++------ .../daily-cache-strategy-analyzer.lock.yml | 25 +++++++++++++------ .../daily-caveman-optimizer.lock.yml | 25 +++++++++++++------ .github/workflows/daily-choice-test.lock.yml | 25 +++++++++++++------ .../workflows/daily-cli-performance.lock.yml | 25 +++++++++++++------ .../workflows/daily-cli-tools-tester.lock.yml | 25 +++++++++++++------ .github/workflows/daily-code-metrics.lock.yml | 25 +++++++++++++------ .../daily-community-attribution.lock.yml | 25 +++++++++++++------ .../workflows/daily-compiler-quality.lock.yml | 25 +++++++++++++------ ...ly-compiler-threat-spec-optimizer.lock.yml | 25 +++++++++++++------ .../daily-credit-limit-test.lock.yml | 25 +++++++++++++------ .github/workflows/daily-doc-healer.lock.yml | 25 +++++++++++++------ .github/workflows/daily-doc-updater.lock.yml | 25 +++++++++++++------ .../daily-experiment-report.lock.yml | 25 +++++++++++++------ .github/workflows/daily-fact.lock.yml | 25 +++++++++++++------ .github/workflows/daily-file-diet.lock.yml | 25 +++++++++++++------ .../workflows/daily-firewall-report.lock.yml | 25 +++++++++++++------ .../daily-formal-spec-verifier.lock.yml | 25 +++++++++++++------ .../workflows/daily-function-namer.lock.yml | 25 +++++++++++++------ .../workflows/daily-geo-optimizer.lock.yml | 25 +++++++++++++------ .github/workflows/daily-hippo-learn.lock.yml | 25 +++++++++++++------ .../workflows/daily-issues-report.lock.yml | 25 +++++++++++++------ .../daily-malicious-code-scan.lock.yml | 25 +++++++++++++------ .../daily-max-ai-credits-test.lock.yml | 25 +++++++++++++------ .../daily-mcp-concurrency-analysis.lock.yml | 25 +++++++++++++------ .../workflows/daily-model-inventory.lock.yml | 25 +++++++++++++------ .../daily-multi-device-docs-tester.lock.yml | 25 +++++++++++++------ .github/workflows/daily-news.lock.yml | 25 +++++++++++++------ .../daily-observability-report.lock.yml | 25 +++++++++++++------ .../daily-performance-summary.lock.yml | 25 +++++++++++++------ .github/workflows/daily-regulatory.lock.yml | 25 +++++++++++++------ .../daily-reliability-review.lock.yml | 25 +++++++++++++------ .../daily-rendering-scripts-verifier.lock.yml | 25 +++++++++++++------ .../workflows/daily-repo-chronicle.lock.yml | 25 +++++++++++++------ .../daily-safe-output-integrator.lock.yml | 25 +++++++++++++------ .../daily-safe-output-optimizer.lock.yml | 25 +++++++++++++------ .../daily-safe-outputs-conformance.lock.yml | 25 +++++++++++++------ .../daily-safeoutputs-git-simulator.lock.yml | 25 +++++++++++++------ .../workflows/daily-secrets-analysis.lock.yml | 25 +++++++++++++------ .../daily-security-observability.lock.yml | 25 +++++++++++++------ .../daily-security-red-team.lock.yml | 25 +++++++++++++------ .github/workflows/daily-semgrep-scan.lock.yml | 25 +++++++++++++------ .../workflows/daily-sentrux-report.lock.yml | 25 +++++++++++++------ .../workflows/daily-skill-optimizer.lock.yml | 25 +++++++++++++------ .../daily-spdd-spec-planner.lock.yml | 25 +++++++++++++------ .../daily-syntax-error-quality.lock.yml | 25 +++++++++++++------ .../daily-team-evolution-insights.lock.yml | 25 +++++++++++++------ .github/workflows/daily-team-status.lock.yml | 25 +++++++++++++------ .../daily-testify-uber-super-expert.lock.yml | 25 +++++++++++++------ .../daily-token-consumption-report.lock.yml | 25 +++++++++++++------ ...dows-terminal-integration-builder.lock.yml | 25 +++++++++++++------ .../workflows/daily-workflow-updater.lock.yml | 25 +++++++++++++------ .../dataflow-pr-discussion-dataset.lock.yml | 25 +++++++++++++------ .github/workflows/dead-code-remover.lock.yml | 25 +++++++++++++------ .github/workflows/deep-report.lock.yml | 25 +++++++++++++------ .github/workflows/delight.lock.yml | 25 +++++++++++++------ .github/workflows/dependabot-burner.lock.yml | 25 +++++++++++++------ .../workflows/dependabot-go-checker.lock.yml | 25 +++++++++++++------ .github/workflows/dependabot-repair.lock.yml | 25 +++++++++++++------ .../deployment-incident-monitor.lock.yml | 25 +++++++++++++------ .../workflows/design-decision-gate.lock.yml | 25 +++++++++++++------ .../workflows/designer-drift-audit.lock.yml | 25 +++++++++++++------ .github/workflows/dev-hawk.lock.yml | 25 +++++++++++++------ .github/workflows/dev.lock.yml | 25 +++++++++++++------ .../developer-docs-consolidator.lock.yml | 25 +++++++++++++------ .github/workflows/dictation-prompt.lock.yml | 25 +++++++++++++------ .../workflows/discussion-task-miner.lock.yml | 25 +++++++++++++------ .github/workflows/docs-noob-tester.lock.yml | 25 +++++++++++++------ .github/workflows/draft-pr-cleanup.lock.yml | 25 +++++++++++++------ .../duplicate-code-detector.lock.yml | 25 +++++++++++++------ .../example-failure-category-filter.lock.yml | 25 +++++++++++++------ .../example-permissions-warning.lock.yml | 25 +++++++++++++------ .../example-workflow-analyzer.lock.yml | 25 +++++++++++++------ .github/workflows/firewall-escape.lock.yml | 25 +++++++++++++------ .github/workflows/firewall.lock.yml | 25 +++++++++++++------ .../workflows/functional-pragmatist.lock.yml | 25 +++++++++++++------ .../github-mcp-structural-analysis.lock.yml | 25 +++++++++++++------ .../github-mcp-tools-report.lock.yml | 25 +++++++++++++------ .../github-remote-mcp-auth-test.lock.yml | 25 +++++++++++++------ .../workflows/glossary-maintainer.lock.yml | 25 +++++++++++++------ .github/workflows/go-fan.lock.yml | 25 +++++++++++++------ .github/workflows/go-logger.lock.yml | 25 +++++++++++++------ .../workflows/go-pattern-detector.lock.yml | 25 +++++++++++++------ .github/workflows/gpclean.lock.yml | 25 +++++++++++++------ .github/workflows/grumpy-reviewer.lock.yml | 25 +++++++++++++------ .github/workflows/hippo-embed.lock.yml | 25 +++++++++++++------ .github/workflows/hourly-ci-cleaner.lock.yml | 25 +++++++++++++------ .../workflows/instructions-janitor.lock.yml | 25 +++++++++++++------ .github/workflows/issue-arborist.lock.yml | 25 +++++++++++++------ .github/workflows/issue-monster.lock.yml | 25 +++++++++++++------ .github/workflows/issue-triage-agent.lock.yml | 25 +++++++++++++------ .github/workflows/jsweep.lock.yml | 25 +++++++++++++------ .../workflows/layout-spec-maintainer.lock.yml | 25 +++++++++++++------ .github/workflows/lint-monster.lock.yml | 25 +++++++++++++------ .github/workflows/linter-miner.lock.yml | 25 +++++++++++++------ .github/workflows/lockfile-stats.lock.yml | 25 +++++++++++++------ .../mattpocock-skills-reviewer.lock.yml | 25 +++++++++++++------ .github/workflows/mcp-inspector.lock.yml | 25 +++++++++++++------ .github/workflows/mergefest.lock.yml | 25 +++++++++++++------ .github/workflows/metrics-collector.lock.yml | 25 +++++++++++++------ .github/workflows/necromancer.lock.yml | 25 +++++++++++++------ .../workflows/notion-issue-summary.lock.yml | 25 +++++++++++++------ .../objective-impact-report.lock.yml | 25 +++++++++++++------ .github/workflows/org-health-report.lock.yml | 25 +++++++++++++------ .github/workflows/outcome-collector.lock.yml | 25 +++++++++++++------ .github/workflows/pdf-summary.lock.yml | 25 +++++++++++++------ .github/workflows/plan.lock.yml | 25 +++++++++++++------ .github/workflows/poem-bot.lock.yml | 25 +++++++++++++------ .github/workflows/portfolio-analyst.lock.yml | 25 +++++++++++++------ .../pr-code-quality-reviewer.lock.yml | 25 +++++++++++++------ .../workflows/pr-description-caveman.lock.yml | 25 +++++++++++++------ .../workflows/pr-nitpick-reviewer.lock.yml | 25 +++++++++++++------ .github/workflows/pr-sous-chef.lock.yml | 25 +++++++++++++------ .github/workflows/pr-triage-agent.lock.yml | 25 +++++++++++++------ .../prompt-clustering-analysis.lock.yml | 25 +++++++++++++------ .github/workflows/python-data-charts.lock.yml | 25 +++++++++++++------ .github/workflows/q.lock.yml | 25 +++++++++++++------ .../workflows/refactoring-cadence.lock.yml | 25 +++++++++++++------ .github/workflows/refiner.lock.yml | 25 +++++++++++++------ .github/workflows/release.lock.yml | 25 +++++++++++++------ .../workflows/repo-audit-analyzer.lock.yml | 25 +++++++++++++------ .github/workflows/repo-tree-map.lock.yml | 25 +++++++++++++------ .../repository-quality-improver.lock.yml | 25 +++++++++++++------ .github/workflows/research.lock.yml | 25 +++++++++++++------ .github/workflows/ruflo-backed-task.lock.yml | 25 +++++++++++++------ .github/workflows/safe-output-health.lock.yml | 25 +++++++++++++------ .../schema-consistency-checker.lock.yml | 25 +++++++++++++------ .../schema-feature-coverage.lock.yml | 25 +++++++++++++------ .github/workflows/scout.lock.yml | 25 +++++++++++++------ .../workflows/security-compliance.lock.yml | 25 +++++++++++++------ .github/workflows/security-review.lock.yml | 25 +++++++++++++------ .../semantic-function-refactor.lock.yml | 25 +++++++++++++------ .github/workflows/sergo.lock.yml | 25 +++++++++++++------ .github/workflows/skillet.lock.yml | 25 +++++++++++++------ .../workflows/slide-deck-maintainer.lock.yml | 25 +++++++++++++------ .../workflows/smoke-agent-all-merged.lock.yml | 25 +++++++++++++------ .../workflows/smoke-agent-all-none.lock.yml | 25 +++++++++++++------ .../smoke-agent-public-approved.lock.yml | 25 +++++++++++++------ .../smoke-agent-public-none.lock.yml | 25 +++++++++++++------ .../smoke-agent-scoped-approved.lock.yml | 25 +++++++++++++------ .github/workflows/smoke-antigravity.lock.yml | 25 +++++++++++++------ .../workflows/smoke-call-workflow.lock.yml | 25 +++++++++++++------ .github/workflows/smoke-ci.lock.yml | 25 +++++++++++++------ .github/workflows/smoke-claude.lock.yml | 25 +++++++++++++------ .github/workflows/smoke-codex.lock.yml | 25 +++++++++++++------ .../smoke-copilot-aoai-apikey.lock.yml | 25 +++++++++++++------ .../smoke-copilot-aoai-entra.lock.yml | 25 +++++++++++++------ .github/workflows/smoke-copilot-arm.lock.yml | 25 +++++++++++++------ .github/workflows/smoke-copilot-sdk.lock.yml | 25 +++++++++++++------ .github/workflows/smoke-copilot.lock.yml | 25 +++++++++++++------ .../smoke-create-cross-repo-pr.lock.yml | 25 +++++++++++++------ .github/workflows/smoke-crush.lock.yml | 25 +++++++++++++------ .github/workflows/smoke-gemini.lock.yml | 25 +++++++++++++------ .github/workflows/smoke-multi-pr.lock.yml | 25 +++++++++++++------ .github/workflows/smoke-opencode.lock.yml | 25 +++++++++++++------ .../workflows/smoke-otel-backends.lock.yml | 25 +++++++++++++------ .github/workflows/smoke-pi.lock.yml | 25 +++++++++++++------ .github/workflows/smoke-project.lock.yml | 25 +++++++++++++------ .../workflows/smoke-service-ports.lock.yml | 25 +++++++++++++------ .github/workflows/smoke-temporary-id.lock.yml | 25 +++++++++++++------ .github/workflows/smoke-test-tools.lock.yml | 25 +++++++++++++------ .../smoke-update-cross-repo-pr.lock.yml | 25 +++++++++++++------ .../smoke-workflow-call-with-inputs.lock.yml | 25 +++++++++++++------ .../workflows/smoke-workflow-call.lock.yml | 25 +++++++++++++------ .github/workflows/spec-enforcer.lock.yml | 25 +++++++++++++------ .github/workflows/spec-extractor.lock.yml | 25 +++++++++++++------ .github/workflows/spec-librarian.lock.yml | 25 +++++++++++++------ .github/workflows/stale-pr-cleanup.lock.yml | 25 +++++++++++++------ .../workflows/stale-repo-identifier.lock.yml | 25 +++++++++++++------ .../workflows/static-analysis-report.lock.yml | 25 +++++++++++++------ .../workflows/step-name-alignment.lock.yml | 25 +++++++++++++------ .github/workflows/sub-issue-closer.lock.yml | 25 +++++++++++++------ .github/workflows/super-linter.lock.yml | 25 +++++++++++++------ .../workflows/technical-doc-writer.lock.yml | 25 +++++++++++++------ .github/workflows/terminal-stylist.lock.yml | 25 +++++++++++++------ .../test-create-pr-error-handling.lock.yml | 25 +++++++++++++------ .github/workflows/test-dispatcher.lock.yml | 25 +++++++++++++------ .../test-project-url-default.lock.yml | 25 +++++++++++++------ .../workflows/test-quality-sentinel.lock.yml | 25 +++++++++++++------ .github/workflows/test-workflow.lock.yml | 25 +++++++++++++------ .github/workflows/tidy.lock.yml | 25 +++++++++++++------ .github/workflows/typist.lock.yml | 25 +++++++++++++------ .../workflows/ubuntu-image-analyzer.lock.yml | 25 +++++++++++++------ .../uk-ai-operational-resilience.lock.yml | 25 +++++++++++++------ .github/workflows/unbloat-docs.lock.yml | 25 +++++++++++++------ .github/workflows/update-astro.lock.yml | 25 +++++++++++++------ .github/workflows/video-analyzer.lock.yml | 25 +++++++++++++------ .../visual-regression-checker.lock.yml | 25 +++++++++++++------ .../weekly-blog-post-writer.lock.yml | 25 +++++++++++++------ .../weekly-editors-health-check.lock.yml | 25 +++++++++++++------ .../workflows/weekly-issue-summary.lock.yml | 25 +++++++++++++------ .../weekly-safe-outputs-spec-review.lock.yml | 25 +++++++++++++------ .github/workflows/workflow-generator.lock.yml | 25 +++++++++++++------ .../workflow-health-manager.lock.yml | 25 +++++++++++++------ .../workflows/workflow-normalizer.lock.yml | 25 +++++++++++++------ .../workflow-skill-extractor.lock.yml | 25 +++++++++++++------ 249 files changed, 4233 insertions(+), 1992 deletions(-) diff --git a/.github/workflows/ab-testing-advisor.lock.yml b/.github/workflows/ab-testing-advisor.lock.yml index 1ef169c6591..ac6198ab0ea 100644 --- a/.github/workflows/ab-testing-advisor.lock.yml +++ b/.github/workflows/ab-testing-advisor.lock.yml @@ -1098,12 +1098,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1130,6 +1131,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1160,7 +1165,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1228,12 +1233,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1296,6 +1304,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/ace-editor.lock.yml b/.github/workflows/ace-editor.lock.yml index 609b12efa48..9b0cf11bd66 100644 --- a/.github/workflows/ace-editor.lock.yml +++ b/.github/workflows/ace-editor.lock.yml @@ -1036,12 +1036,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1068,6 +1069,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1098,7 +1103,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1166,12 +1171,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1234,6 +1242,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/agent-performance-analyzer.lock.yml b/.github/workflows/agent-performance-analyzer.lock.yml index 759282e189a..52f5df47152 100644 --- a/.github/workflows/agent-performance-analyzer.lock.yml +++ b/.github/workflows/agent-performance-analyzer.lock.yml @@ -1305,12 +1305,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1337,6 +1338,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1367,7 +1372,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1435,12 +1440,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1503,6 +1511,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/agent-persona-explorer.lock.yml b/.github/workflows/agent-persona-explorer.lock.yml index f94056d6b2d..c33c384ab5a 100644 --- a/.github/workflows/agent-persona-explorer.lock.yml +++ b/.github/workflows/agent-persona-explorer.lock.yml @@ -1222,12 +1222,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1254,6 +1255,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1284,7 +1289,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1352,12 +1357,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1420,6 +1428,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/agentic-token-audit.lock.yml b/.github/workflows/agentic-token-audit.lock.yml index 1fe4ea0875e..9454917207d 100644 --- a/.github/workflows/agentic-token-audit.lock.yml +++ b/.github/workflows/agentic-token-audit.lock.yml @@ -1236,12 +1236,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1268,6 +1269,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1298,7 +1303,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1366,12 +1371,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1434,6 +1442,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/agentic-token-optimizer.lock.yml b/.github/workflows/agentic-token-optimizer.lock.yml index 0f76d6c1e3d..c6cd5563e04 100644 --- a/.github/workflows/agentic-token-optimizer.lock.yml +++ b/.github/workflows/agentic-token-optimizer.lock.yml @@ -1103,12 +1103,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1135,6 +1136,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1165,7 +1170,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1233,12 +1238,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1301,6 +1309,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/agentic-token-trend-audit.lock.yml b/.github/workflows/agentic-token-trend-audit.lock.yml index b8014258f95..801121518b6 100644 --- a/.github/workflows/agentic-token-trend-audit.lock.yml +++ b/.github/workflows/agentic-token-trend-audit.lock.yml @@ -1193,12 +1193,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1225,6 +1226,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1255,7 +1260,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1323,12 +1328,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1391,6 +1399,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/ai-moderator.lock.yml b/.github/workflows/ai-moderator.lock.yml index f25d22461ee..ad5e35b39e9 100644 --- a/.github/workflows/ai-moderator.lock.yml +++ b/.github/workflows/ai-moderator.lock.yml @@ -1226,12 +1226,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1258,6 +1259,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1288,7 +1293,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1356,12 +1361,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1424,6 +1432,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/api-consumption-report.lock.yml b/.github/workflows/api-consumption-report.lock.yml index 8071fb733e2..e54ff0dd993 100644 --- a/.github/workflows/api-consumption-report.lock.yml +++ b/.github/workflows/api-consumption-report.lock.yml @@ -1575,12 +1575,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1607,6 +1608,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1637,7 +1642,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1705,12 +1710,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1773,6 +1781,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/approach-validator.lock.yml b/.github/workflows/approach-validator.lock.yml index 0f44031a94c..e89c160a811 100644 --- a/.github/workflows/approach-validator.lock.yml +++ b/.github/workflows/approach-validator.lock.yml @@ -1279,12 +1279,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1311,6 +1312,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1341,7 +1346,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1409,12 +1414,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1477,6 +1485,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/archie.lock.yml b/.github/workflows/archie.lock.yml index 2b332d9bf81..6eb5c1b0b19 100644 --- a/.github/workflows/archie.lock.yml +++ b/.github/workflows/archie.lock.yml @@ -1164,12 +1164,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1196,6 +1197,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1226,7 +1231,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1294,12 +1299,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1362,6 +1370,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/architecture-guardian.lock.yml b/.github/workflows/architecture-guardian.lock.yml index 4c9283ce36e..cbf1c6a64f7 100644 --- a/.github/workflows/architecture-guardian.lock.yml +++ b/.github/workflows/architecture-guardian.lock.yml @@ -1172,12 +1172,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1204,6 +1205,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1234,7 +1239,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1302,12 +1307,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1370,6 +1378,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/artifacts-summary.lock.yml b/.github/workflows/artifacts-summary.lock.yml index d4bce9a856b..9cfd8bd74b1 100644 --- a/.github/workflows/artifacts-summary.lock.yml +++ b/.github/workflows/artifacts-summary.lock.yml @@ -1078,12 +1078,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1110,6 +1111,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1140,7 +1145,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1208,12 +1213,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1276,6 +1284,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/audit-workflows.lock.yml b/.github/workflows/audit-workflows.lock.yml index b2545e74d19..91ef95d2ced 100644 --- a/.github/workflows/audit-workflows.lock.yml +++ b/.github/workflows/audit-workflows.lock.yml @@ -1372,12 +1372,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1404,6 +1405,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1434,7 +1439,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1502,12 +1507,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1570,6 +1578,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/auto-triage-issues.lock.yml b/.github/workflows/auto-triage-issues.lock.yml index a584a17506a..2f577c402bb 100644 --- a/.github/workflows/auto-triage-issues.lock.yml +++ b/.github/workflows/auto-triage-issues.lock.yml @@ -1105,12 +1105,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1137,6 +1138,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1167,7 +1172,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1235,12 +1240,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1303,6 +1311,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/avenger.lock.yml b/.github/workflows/avenger.lock.yml index b16468c7c83..acc63d9d075 100644 --- a/.github/workflows/avenger.lock.yml +++ b/.github/workflows/avenger.lock.yml @@ -1262,12 +1262,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1294,6 +1295,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1324,7 +1329,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1392,12 +1397,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1460,6 +1468,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/aw-failure-investigator.lock.yml b/.github/workflows/aw-failure-investigator.lock.yml index d91a7470658..ffe0666ebd0 100644 --- a/.github/workflows/aw-failure-investigator.lock.yml +++ b/.github/workflows/aw-failure-investigator.lock.yml @@ -1363,12 +1363,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1395,6 +1396,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1425,7 +1430,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1493,12 +1498,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1561,6 +1569,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/blog-auditor.lock.yml b/.github/workflows/blog-auditor.lock.yml index 2bd9b4f43ad..04185b47dba 100644 --- a/.github/workflows/blog-auditor.lock.yml +++ b/.github/workflows/blog-auditor.lock.yml @@ -1241,12 +1241,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1273,6 +1274,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1303,7 +1308,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1371,12 +1376,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1439,6 +1447,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/bot-detection.lock.yml b/.github/workflows/bot-detection.lock.yml index af64afd1c66..0baa7fb6b88 100644 --- a/.github/workflows/bot-detection.lock.yml +++ b/.github/workflows/bot-detection.lock.yml @@ -1166,12 +1166,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1198,6 +1199,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1228,7 +1233,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1296,12 +1301,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1364,6 +1372,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/brave.lock.yml b/.github/workflows/brave.lock.yml index 30f21ebabbf..18f1ad656e5 100644 --- a/.github/workflows/brave.lock.yml +++ b/.github/workflows/brave.lock.yml @@ -1160,12 +1160,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1192,6 +1193,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1222,7 +1227,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1290,12 +1295,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1358,6 +1366,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/breaking-change-checker.lock.yml b/.github/workflows/breaking-change-checker.lock.yml index a5aa44c973c..63636ece599 100644 --- a/.github/workflows/breaking-change-checker.lock.yml +++ b/.github/workflows/breaking-change-checker.lock.yml @@ -1120,12 +1120,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1152,6 +1153,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1182,7 +1187,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1250,12 +1255,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1318,6 +1326,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/changeset.lock.yml b/.github/workflows/changeset.lock.yml index 5a2d8a940a5..ec0bb45707e 100644 --- a/.github/workflows/changeset.lock.yml +++ b/.github/workflows/changeset.lock.yml @@ -1207,12 +1207,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1239,6 +1240,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1269,7 +1274,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1337,12 +1342,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1405,6 +1413,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/chaos-pr-bundle-fuzzer.lock.yml b/.github/workflows/chaos-pr-bundle-fuzzer.lock.yml index 7583cfbe7e5..2a645372401 100644 --- a/.github/workflows/chaos-pr-bundle-fuzzer.lock.yml +++ b/.github/workflows/chaos-pr-bundle-fuzzer.lock.yml @@ -1104,12 +1104,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1136,6 +1137,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1166,7 +1171,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1234,12 +1239,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1302,6 +1310,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/ci-coach.lock.yml b/.github/workflows/ci-coach.lock.yml index 1cdcd80e6d0..6d441b5cbca 100644 --- a/.github/workflows/ci-coach.lock.yml +++ b/.github/workflows/ci-coach.lock.yml @@ -1214,12 +1214,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1246,6 +1247,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1276,7 +1281,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1344,12 +1349,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1412,6 +1420,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/ci-doctor.lock.yml b/.github/workflows/ci-doctor.lock.yml index e86163dbb52..d67f0a5f4a1 100644 --- a/.github/workflows/ci-doctor.lock.yml +++ b/.github/workflows/ci-doctor.lock.yml @@ -1383,12 +1383,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1415,6 +1416,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1445,7 +1450,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1513,12 +1518,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1581,6 +1589,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/claude-code-user-docs-review.lock.yml b/.github/workflows/claude-code-user-docs-review.lock.yml index 237c04d3011..b07f898a232 100644 --- a/.github/workflows/claude-code-user-docs-review.lock.yml +++ b/.github/workflows/claude-code-user-docs-review.lock.yml @@ -1209,12 +1209,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1241,6 +1242,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1271,7 +1276,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1339,12 +1344,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1407,6 +1415,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/cli-consistency-checker.lock.yml b/.github/workflows/cli-consistency-checker.lock.yml index 4d760ec33c0..3a4b4722f23 100644 --- a/.github/workflows/cli-consistency-checker.lock.yml +++ b/.github/workflows/cli-consistency-checker.lock.yml @@ -1093,12 +1093,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1125,6 +1126,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1155,7 +1160,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1223,12 +1228,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1291,6 +1299,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/cli-version-checker.lock.yml b/.github/workflows/cli-version-checker.lock.yml index 2892c8153e8..d9d1838f43e 100644 --- a/.github/workflows/cli-version-checker.lock.yml +++ b/.github/workflows/cli-version-checker.lock.yml @@ -1203,12 +1203,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1235,6 +1236,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1265,7 +1270,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1333,12 +1338,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1401,6 +1409,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/cloclo.lock.yml b/.github/workflows/cloclo.lock.yml index 055ea17cdf7..87f510a0290 100644 --- a/.github/workflows/cloclo.lock.yml +++ b/.github/workflows/cloclo.lock.yml @@ -1490,12 +1490,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1522,6 +1523,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1552,7 +1557,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1620,12 +1625,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1688,6 +1696,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/code-scanning-fixer.lock.yml b/.github/workflows/code-scanning-fixer.lock.yml index 5699a9d0740..526345345db 100644 --- a/.github/workflows/code-scanning-fixer.lock.yml +++ b/.github/workflows/code-scanning-fixer.lock.yml @@ -1201,12 +1201,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1233,6 +1234,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1263,7 +1268,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1331,12 +1336,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1399,6 +1407,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/code-simplifier.lock.yml b/.github/workflows/code-simplifier.lock.yml index 9969893d902..141f38c40c0 100644 --- a/.github/workflows/code-simplifier.lock.yml +++ b/.github/workflows/code-simplifier.lock.yml @@ -1157,12 +1157,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1189,6 +1190,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1219,7 +1224,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1287,12 +1292,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1355,6 +1363,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/codex-github-remote-mcp-test.lock.yml b/.github/workflows/codex-github-remote-mcp-test.lock.yml index 2555ec70149..4ab33e0877f 100644 --- a/.github/workflows/codex-github-remote-mcp-test.lock.yml +++ b/.github/workflows/codex-github-remote-mcp-test.lock.yml @@ -1041,12 +1041,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1073,6 +1074,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1103,7 +1108,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1171,12 +1176,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1239,6 +1247,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/commit-changes-analyzer.lock.yml b/.github/workflows/commit-changes-analyzer.lock.yml index 420a49aa34b..d63c3a709d5 100644 --- a/.github/workflows/commit-changes-analyzer.lock.yml +++ b/.github/workflows/commit-changes-analyzer.lock.yml @@ -1049,12 +1049,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1081,6 +1082,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1111,7 +1116,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1179,12 +1184,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1247,6 +1255,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/constraint-solving-potd.lock.yml b/.github/workflows/constraint-solving-potd.lock.yml index 587a99c4adf..b76fdece349 100644 --- a/.github/workflows/constraint-solving-potd.lock.yml +++ b/.github/workflows/constraint-solving-potd.lock.yml @@ -1103,12 +1103,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1135,6 +1136,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1165,7 +1170,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1233,12 +1238,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1301,6 +1309,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/contribution-check.lock.yml b/.github/workflows/contribution-check.lock.yml index 699509c85bb..098cf97a357 100644 --- a/.github/workflows/contribution-check.lock.yml +++ b/.github/workflows/contribution-check.lock.yml @@ -1216,12 +1216,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1248,6 +1249,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1278,7 +1283,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1346,12 +1351,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1414,6 +1422,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/copilot-agent-analysis.lock.yml b/.github/workflows/copilot-agent-analysis.lock.yml index 3f99cd0afcd..3be5db6bc52 100644 --- a/.github/workflows/copilot-agent-analysis.lock.yml +++ b/.github/workflows/copilot-agent-analysis.lock.yml @@ -1286,12 +1286,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1318,6 +1319,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1348,7 +1353,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1416,12 +1421,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1484,6 +1492,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/copilot-centralization-drilldown.lock.yml b/.github/workflows/copilot-centralization-drilldown.lock.yml index 2902f3e9bf3..aadac057bdb 100644 --- a/.github/workflows/copilot-centralization-drilldown.lock.yml +++ b/.github/workflows/copilot-centralization-drilldown.lock.yml @@ -1066,12 +1066,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1098,6 +1099,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1128,7 +1133,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1196,12 +1201,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1264,6 +1272,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/copilot-centralization-optimizer.lock.yml b/.github/workflows/copilot-centralization-optimizer.lock.yml index 86f94f9129e..6f4ce489b8f 100644 --- a/.github/workflows/copilot-centralization-optimizer.lock.yml +++ b/.github/workflows/copilot-centralization-optimizer.lock.yml @@ -1114,12 +1114,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1146,6 +1147,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1176,7 +1181,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1244,12 +1249,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1312,6 +1320,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/copilot-cli-deep-research.lock.yml b/.github/workflows/copilot-cli-deep-research.lock.yml index 686708f88b6..68a7e28de60 100644 --- a/.github/workflows/copilot-cli-deep-research.lock.yml +++ b/.github/workflows/copilot-cli-deep-research.lock.yml @@ -1121,12 +1121,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1153,6 +1154,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1183,7 +1188,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1251,12 +1256,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1319,6 +1327,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/copilot-opt.lock.yml b/.github/workflows/copilot-opt.lock.yml index 1abe808a851..fd66c073e4e 100644 --- a/.github/workflows/copilot-opt.lock.yml +++ b/.github/workflows/copilot-opt.lock.yml @@ -1191,12 +1191,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1223,6 +1224,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1253,7 +1258,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1321,12 +1326,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1389,6 +1397,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/copilot-pr-merged-report.lock.yml b/.github/workflows/copilot-pr-merged-report.lock.yml index 45edabe1356..7b68cd299f8 100644 --- a/.github/workflows/copilot-pr-merged-report.lock.yml +++ b/.github/workflows/copilot-pr-merged-report.lock.yml @@ -1059,12 +1059,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1091,6 +1092,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1121,7 +1126,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1189,12 +1194,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1257,6 +1265,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/copilot-pr-nlp-analysis.lock.yml b/.github/workflows/copilot-pr-nlp-analysis.lock.yml index 2188e2531fd..94435c4165e 100644 --- a/.github/workflows/copilot-pr-nlp-analysis.lock.yml +++ b/.github/workflows/copilot-pr-nlp-analysis.lock.yml @@ -1247,12 +1247,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1279,6 +1280,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1309,7 +1314,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1377,12 +1382,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1445,6 +1453,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/copilot-pr-prompt-analysis.lock.yml b/.github/workflows/copilot-pr-prompt-analysis.lock.yml index 2ee4fe03a56..30777759f7a 100644 --- a/.github/workflows/copilot-pr-prompt-analysis.lock.yml +++ b/.github/workflows/copilot-pr-prompt-analysis.lock.yml @@ -1186,12 +1186,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1218,6 +1219,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1248,7 +1253,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1316,12 +1321,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1384,6 +1392,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/copilot-session-insights.lock.yml b/.github/workflows/copilot-session-insights.lock.yml index 1666a20c518..7a4b58147bd 100644 --- a/.github/workflows/copilot-session-insights.lock.yml +++ b/.github/workflows/copilot-session-insights.lock.yml @@ -1305,12 +1305,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1337,6 +1338,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1367,7 +1372,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1435,12 +1440,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1503,6 +1511,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/craft.lock.yml b/.github/workflows/craft.lock.yml index e1c64216975..54d1bbbaa4e 100644 --- a/.github/workflows/craft.lock.yml +++ b/.github/workflows/craft.lock.yml @@ -1161,12 +1161,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1193,6 +1194,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1223,7 +1228,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1291,12 +1296,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1359,6 +1367,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-agent-of-the-day-blog-writer.lock.yml b/.github/workflows/daily-agent-of-the-day-blog-writer.lock.yml index a338cc3cef9..9c249defddf 100644 --- a/.github/workflows/daily-agent-of-the-day-blog-writer.lock.yml +++ b/.github/workflows/daily-agent-of-the-day-blog-writer.lock.yml @@ -1294,12 +1294,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1326,6 +1327,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1356,7 +1361,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1424,12 +1429,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1492,6 +1500,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-agentrx-trace-optimizer.lock.yml b/.github/workflows/daily-agentrx-trace-optimizer.lock.yml index 5f73655a085..1bcfe1756b0 100644 --- a/.github/workflows/daily-agentrx-trace-optimizer.lock.yml +++ b/.github/workflows/daily-agentrx-trace-optimizer.lock.yml @@ -1310,12 +1310,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1342,6 +1343,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1372,7 +1377,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1440,12 +1445,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1508,6 +1516,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-ambient-context-optimizer.lock.yml b/.github/workflows/daily-ambient-context-optimizer.lock.yml index 8849f0ce08d..fd24e4a847f 100644 --- a/.github/workflows/daily-ambient-context-optimizer.lock.yml +++ b/.github/workflows/daily-ambient-context-optimizer.lock.yml @@ -1179,12 +1179,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1211,6 +1212,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1241,7 +1246,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1309,12 +1314,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1377,6 +1385,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-architecture-diagram.lock.yml b/.github/workflows/daily-architecture-diagram.lock.yml index af59e3c7939..f75d6112df7 100644 --- a/.github/workflows/daily-architecture-diagram.lock.yml +++ b/.github/workflows/daily-architecture-diagram.lock.yml @@ -1251,12 +1251,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1283,6 +1284,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1313,7 +1318,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1381,12 +1386,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1449,6 +1457,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-assign-issue-to-user.lock.yml b/.github/workflows/daily-assign-issue-to-user.lock.yml index 7ba5255e15a..7cbd53febdc 100644 --- a/.github/workflows/daily-assign-issue-to-user.lock.yml +++ b/.github/workflows/daily-assign-issue-to-user.lock.yml @@ -1090,12 +1090,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1122,6 +1123,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1152,7 +1157,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1220,12 +1225,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1288,6 +1296,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-astrostylelite-markdown-spellcheck.lock.yml b/.github/workflows/daily-astrostylelite-markdown-spellcheck.lock.yml index a28db65c39a..eb15157ebe1 100644 --- a/.github/workflows/daily-astrostylelite-markdown-spellcheck.lock.yml +++ b/.github/workflows/daily-astrostylelite-markdown-spellcheck.lock.yml @@ -1204,12 +1204,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1236,6 +1237,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1266,7 +1271,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1334,12 +1339,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1402,6 +1410,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-aw-cross-repo-compile-check.lock.yml b/.github/workflows/daily-aw-cross-repo-compile-check.lock.yml index 5eb237bafd5..6e46be7c795 100644 --- a/.github/workflows/daily-aw-cross-repo-compile-check.lock.yml +++ b/.github/workflows/daily-aw-cross-repo-compile-check.lock.yml @@ -1199,12 +1199,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1231,6 +1232,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1261,7 +1266,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1329,12 +1334,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1397,6 +1405,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-awf-spec-compiler-surfacing.lock.yml b/.github/workflows/daily-awf-spec-compiler-surfacing.lock.yml index e6d342602fc..7570890be11 100644 --- a/.github/workflows/daily-awf-spec-compiler-surfacing.lock.yml +++ b/.github/workflows/daily-awf-spec-compiler-surfacing.lock.yml @@ -1092,12 +1092,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1124,6 +1125,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1154,7 +1159,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1222,12 +1227,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1290,6 +1298,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-byok-ollama-test.lock.yml b/.github/workflows/daily-byok-ollama-test.lock.yml index 0d30cf05b5c..059803637e6 100644 --- a/.github/workflows/daily-byok-ollama-test.lock.yml +++ b/.github/workflows/daily-byok-ollama-test.lock.yml @@ -1069,12 +1069,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1101,6 +1102,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1131,7 +1136,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1199,12 +1204,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1267,6 +1275,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-cache-strategy-analyzer.lock.yml b/.github/workflows/daily-cache-strategy-analyzer.lock.yml index a137774dfd4..313474d98dd 100644 --- a/.github/workflows/daily-cache-strategy-analyzer.lock.yml +++ b/.github/workflows/daily-cache-strategy-analyzer.lock.yml @@ -1337,12 +1337,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1369,6 +1370,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1399,7 +1404,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1467,12 +1472,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1535,6 +1543,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-caveman-optimizer.lock.yml b/.github/workflows/daily-caveman-optimizer.lock.yml index a2d54fc0c0c..910eaf2b7a1 100644 --- a/.github/workflows/daily-caveman-optimizer.lock.yml +++ b/.github/workflows/daily-caveman-optimizer.lock.yml @@ -1242,12 +1242,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1274,6 +1275,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1304,7 +1309,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1372,12 +1377,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1440,6 +1448,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-choice-test.lock.yml b/.github/workflows/daily-choice-test.lock.yml index b502119cc83..fbec61e5377 100644 --- a/.github/workflows/daily-choice-test.lock.yml +++ b/.github/workflows/daily-choice-test.lock.yml @@ -1137,12 +1137,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1169,6 +1170,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1199,7 +1204,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1267,12 +1272,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1335,6 +1343,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-cli-performance.lock.yml b/.github/workflows/daily-cli-performance.lock.yml index 20f75d51c16..15a0865b509 100644 --- a/.github/workflows/daily-cli-performance.lock.yml +++ b/.github/workflows/daily-cli-performance.lock.yml @@ -1373,12 +1373,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1405,6 +1406,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1435,7 +1440,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1503,12 +1508,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1571,6 +1579,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-cli-tools-tester.lock.yml b/.github/workflows/daily-cli-tools-tester.lock.yml index 4ef4e0ea92c..846de22b381 100644 --- a/.github/workflows/daily-cli-tools-tester.lock.yml +++ b/.github/workflows/daily-cli-tools-tester.lock.yml @@ -1204,12 +1204,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1236,6 +1237,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1266,7 +1271,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1334,12 +1339,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1402,6 +1410,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-code-metrics.lock.yml b/.github/workflows/daily-code-metrics.lock.yml index fee72eef5e3..72fc0ed6e57 100644 --- a/.github/workflows/daily-code-metrics.lock.yml +++ b/.github/workflows/daily-code-metrics.lock.yml @@ -1323,12 +1323,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1355,6 +1356,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1385,7 +1390,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1453,12 +1458,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1521,6 +1529,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-community-attribution.lock.yml b/.github/workflows/daily-community-attribution.lock.yml index 759ade6db0b..209c78b65a7 100644 --- a/.github/workflows/daily-community-attribution.lock.yml +++ b/.github/workflows/daily-community-attribution.lock.yml @@ -1263,12 +1263,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1295,6 +1296,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1325,7 +1330,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1393,12 +1398,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1461,6 +1469,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-compiler-quality.lock.yml b/.github/workflows/daily-compiler-quality.lock.yml index 60454581a48..82165088496 100644 --- a/.github/workflows/daily-compiler-quality.lock.yml +++ b/.github/workflows/daily-compiler-quality.lock.yml @@ -1239,12 +1239,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1271,6 +1272,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1301,7 +1306,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1369,12 +1374,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1437,6 +1445,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-compiler-threat-spec-optimizer.lock.yml b/.github/workflows/daily-compiler-threat-spec-optimizer.lock.yml index 4add5be50bc..68cc8de18b9 100644 --- a/.github/workflows/daily-compiler-threat-spec-optimizer.lock.yml +++ b/.github/workflows/daily-compiler-threat-spec-optimizer.lock.yml @@ -1164,12 +1164,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1196,6 +1197,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1226,7 +1231,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1294,12 +1299,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1362,6 +1370,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-credit-limit-test.lock.yml b/.github/workflows/daily-credit-limit-test.lock.yml index d640527ea82..ae4fe1d5e8a 100644 --- a/.github/workflows/daily-credit-limit-test.lock.yml +++ b/.github/workflows/daily-credit-limit-test.lock.yml @@ -1047,12 +1047,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1079,6 +1080,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1109,7 +1114,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1177,12 +1182,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1245,6 +1253,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-doc-healer.lock.yml b/.github/workflows/daily-doc-healer.lock.yml index de4484a1f8b..d29218973a1 100644 --- a/.github/workflows/daily-doc-healer.lock.yml +++ b/.github/workflows/daily-doc-healer.lock.yml @@ -1346,12 +1346,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1378,6 +1379,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1408,7 +1413,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1476,12 +1481,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1544,6 +1552,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-doc-updater.lock.yml b/.github/workflows/daily-doc-updater.lock.yml index 51c6693affc..b2bd742b4e8 100644 --- a/.github/workflows/daily-doc-updater.lock.yml +++ b/.github/workflows/daily-doc-updater.lock.yml @@ -1148,12 +1148,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1180,6 +1181,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1210,7 +1215,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1278,12 +1283,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1346,6 +1354,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-experiment-report.lock.yml b/.github/workflows/daily-experiment-report.lock.yml index b7240b599d3..31fd113c906 100644 --- a/.github/workflows/daily-experiment-report.lock.yml +++ b/.github/workflows/daily-experiment-report.lock.yml @@ -1235,12 +1235,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1267,6 +1268,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1297,7 +1302,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1365,12 +1370,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1433,6 +1441,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-fact.lock.yml b/.github/workflows/daily-fact.lock.yml index f8f86031edd..6d1acb2e402 100644 --- a/.github/workflows/daily-fact.lock.yml +++ b/.github/workflows/daily-fact.lock.yml @@ -1350,12 +1350,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1382,6 +1383,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1412,7 +1417,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1480,12 +1485,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1548,6 +1556,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-file-diet.lock.yml b/.github/workflows/daily-file-diet.lock.yml index d2de890c660..a5d446b7751 100644 --- a/.github/workflows/daily-file-diet.lock.yml +++ b/.github/workflows/daily-file-diet.lock.yml @@ -1161,12 +1161,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1193,6 +1194,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1223,7 +1228,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1291,12 +1296,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1359,6 +1367,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-firewall-report.lock.yml b/.github/workflows/daily-firewall-report.lock.yml index b691f538e54..bf5be9c7d91 100644 --- a/.github/workflows/daily-firewall-report.lock.yml +++ b/.github/workflows/daily-firewall-report.lock.yml @@ -1163,12 +1163,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1195,6 +1196,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1225,7 +1230,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1293,12 +1298,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1361,6 +1369,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-formal-spec-verifier.lock.yml b/.github/workflows/daily-formal-spec-verifier.lock.yml index 18a3de2ab90..3c0d10ad518 100644 --- a/.github/workflows/daily-formal-spec-verifier.lock.yml +++ b/.github/workflows/daily-formal-spec-verifier.lock.yml @@ -1207,12 +1207,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1239,6 +1240,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1269,7 +1274,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1337,12 +1342,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1405,6 +1413,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-function-namer.lock.yml b/.github/workflows/daily-function-namer.lock.yml index 6fcaa3dd63b..18a321e8e3c 100644 --- a/.github/workflows/daily-function-namer.lock.yml +++ b/.github/workflows/daily-function-namer.lock.yml @@ -1164,12 +1164,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1196,6 +1197,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1226,7 +1231,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1294,12 +1299,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1362,6 +1370,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-geo-optimizer.lock.yml b/.github/workflows/daily-geo-optimizer.lock.yml index 934a2e1ab53..65ebec557f3 100644 --- a/.github/workflows/daily-geo-optimizer.lock.yml +++ b/.github/workflows/daily-geo-optimizer.lock.yml @@ -1115,12 +1115,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1147,6 +1148,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1177,7 +1182,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1245,12 +1250,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1313,6 +1321,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-hippo-learn.lock.yml b/.github/workflows/daily-hippo-learn.lock.yml index f40e8ae3d5f..59999d8bf94 100644 --- a/.github/workflows/daily-hippo-learn.lock.yml +++ b/.github/workflows/daily-hippo-learn.lock.yml @@ -1218,12 +1218,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1250,6 +1251,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1280,7 +1285,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1348,12 +1353,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1416,6 +1424,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-issues-report.lock.yml b/.github/workflows/daily-issues-report.lock.yml index a2d50dec323..a0e65f7faac 100644 --- a/.github/workflows/daily-issues-report.lock.yml +++ b/.github/workflows/daily-issues-report.lock.yml @@ -1390,12 +1390,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1422,6 +1423,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1452,7 +1457,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1520,12 +1525,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1588,6 +1596,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-malicious-code-scan.lock.yml b/.github/workflows/daily-malicious-code-scan.lock.yml index 9f390b787ac..ab71b41cb3f 100644 --- a/.github/workflows/daily-malicious-code-scan.lock.yml +++ b/.github/workflows/daily-malicious-code-scan.lock.yml @@ -1125,12 +1125,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1157,6 +1158,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1187,7 +1192,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1255,12 +1260,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1323,6 +1331,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-max-ai-credits-test.lock.yml b/.github/workflows/daily-max-ai-credits-test.lock.yml index 0dbd21189e8..81968b10efc 100644 --- a/.github/workflows/daily-max-ai-credits-test.lock.yml +++ b/.github/workflows/daily-max-ai-credits-test.lock.yml @@ -986,12 +986,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1018,6 +1019,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1048,7 +1053,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1116,12 +1121,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1184,6 +1192,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-mcp-concurrency-analysis.lock.yml b/.github/workflows/daily-mcp-concurrency-analysis.lock.yml index 82f3b7b9b6b..c8896b60a9c 100644 --- a/.github/workflows/daily-mcp-concurrency-analysis.lock.yml +++ b/.github/workflows/daily-mcp-concurrency-analysis.lock.yml @@ -1243,12 +1243,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1275,6 +1276,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1305,7 +1310,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1373,12 +1378,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1441,6 +1449,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-model-inventory.lock.yml b/.github/workflows/daily-model-inventory.lock.yml index 4264cd93501..6ace2d03c98 100644 --- a/.github/workflows/daily-model-inventory.lock.yml +++ b/.github/workflows/daily-model-inventory.lock.yml @@ -1433,12 +1433,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1465,6 +1466,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1495,7 +1500,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1563,12 +1568,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1631,6 +1639,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-multi-device-docs-tester.lock.yml b/.github/workflows/daily-multi-device-docs-tester.lock.yml index 34c844fe368..85544e1cbb3 100644 --- a/.github/workflows/daily-multi-device-docs-tester.lock.yml +++ b/.github/workflows/daily-multi-device-docs-tester.lock.yml @@ -1138,12 +1138,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1170,6 +1171,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1200,7 +1205,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1268,12 +1273,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1336,6 +1344,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-news.lock.yml b/.github/workflows/daily-news.lock.yml index fa12f9bf386..043f585aaa5 100644 --- a/.github/workflows/daily-news.lock.yml +++ b/.github/workflows/daily-news.lock.yml @@ -1358,12 +1358,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1390,6 +1391,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1420,7 +1425,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1488,12 +1493,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1556,6 +1564,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-observability-report.lock.yml b/.github/workflows/daily-observability-report.lock.yml index 7d4d0ee7286..63296423eca 100644 --- a/.github/workflows/daily-observability-report.lock.yml +++ b/.github/workflows/daily-observability-report.lock.yml @@ -1209,12 +1209,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1241,6 +1242,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1271,7 +1276,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1339,12 +1344,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1407,6 +1415,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-performance-summary.lock.yml b/.github/workflows/daily-performance-summary.lock.yml index 0aeb9434fe3..a709d96d589 100644 --- a/.github/workflows/daily-performance-summary.lock.yml +++ b/.github/workflows/daily-performance-summary.lock.yml @@ -1673,12 +1673,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1705,6 +1706,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1735,7 +1740,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1803,12 +1808,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1871,6 +1879,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-regulatory.lock.yml b/.github/workflows/daily-regulatory.lock.yml index 29e638edbe8..1bfefa330a6 100644 --- a/.github/workflows/daily-regulatory.lock.yml +++ b/.github/workflows/daily-regulatory.lock.yml @@ -1602,12 +1602,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1634,6 +1635,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1664,7 +1669,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1732,12 +1737,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1800,6 +1808,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-reliability-review.lock.yml b/.github/workflows/daily-reliability-review.lock.yml index b74c53a70f5..9da3ae481d3 100644 --- a/.github/workflows/daily-reliability-review.lock.yml +++ b/.github/workflows/daily-reliability-review.lock.yml @@ -1221,12 +1221,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1253,6 +1254,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1283,7 +1288,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1351,12 +1356,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1419,6 +1427,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-rendering-scripts-verifier.lock.yml b/.github/workflows/daily-rendering-scripts-verifier.lock.yml index 5a532703017..449ab74ee5c 100644 --- a/.github/workflows/daily-rendering-scripts-verifier.lock.yml +++ b/.github/workflows/daily-rendering-scripts-verifier.lock.yml @@ -1374,12 +1374,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1406,6 +1407,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1436,7 +1441,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1504,12 +1509,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1572,6 +1580,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-repo-chronicle.lock.yml b/.github/workflows/daily-repo-chronicle.lock.yml index eafe925194d..d1c9a07c9df 100644 --- a/.github/workflows/daily-repo-chronicle.lock.yml +++ b/.github/workflows/daily-repo-chronicle.lock.yml @@ -1179,12 +1179,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1211,6 +1212,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1241,7 +1246,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1309,12 +1314,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1377,6 +1385,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-safe-output-integrator.lock.yml b/.github/workflows/daily-safe-output-integrator.lock.yml index 4b8d2137f47..7876aa78a75 100644 --- a/.github/workflows/daily-safe-output-integrator.lock.yml +++ b/.github/workflows/daily-safe-output-integrator.lock.yml @@ -1163,12 +1163,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1195,6 +1196,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1225,7 +1230,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1293,12 +1298,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1361,6 +1369,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-safe-output-optimizer.lock.yml b/.github/workflows/daily-safe-output-optimizer.lock.yml index fabbe9e23d4..0aab8f90d87 100644 --- a/.github/workflows/daily-safe-output-optimizer.lock.yml +++ b/.github/workflows/daily-safe-output-optimizer.lock.yml @@ -1395,12 +1395,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1427,6 +1428,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1457,7 +1462,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1525,12 +1530,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1593,6 +1601,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-safe-outputs-conformance.lock.yml b/.github/workflows/daily-safe-outputs-conformance.lock.yml index 9c08aee2d3d..f33761f2ef5 100644 --- a/.github/workflows/daily-safe-outputs-conformance.lock.yml +++ b/.github/workflows/daily-safe-outputs-conformance.lock.yml @@ -1177,12 +1177,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1209,6 +1210,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1239,7 +1244,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1307,12 +1312,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1375,6 +1383,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-safeoutputs-git-simulator.lock.yml b/.github/workflows/daily-safeoutputs-git-simulator.lock.yml index 409a92a4bf8..e2e25be8a58 100644 --- a/.github/workflows/daily-safeoutputs-git-simulator.lock.yml +++ b/.github/workflows/daily-safeoutputs-git-simulator.lock.yml @@ -1236,12 +1236,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1268,6 +1269,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1298,7 +1303,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1366,12 +1371,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1434,6 +1442,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-secrets-analysis.lock.yml b/.github/workflows/daily-secrets-analysis.lock.yml index fa4a927b8df..d5571965124 100644 --- a/.github/workflows/daily-secrets-analysis.lock.yml +++ b/.github/workflows/daily-secrets-analysis.lock.yml @@ -1081,12 +1081,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1113,6 +1114,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1143,7 +1148,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1211,12 +1216,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1279,6 +1287,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-security-observability.lock.yml b/.github/workflows/daily-security-observability.lock.yml index 9b944853945..7d06e9ffa46 100644 --- a/.github/workflows/daily-security-observability.lock.yml +++ b/.github/workflows/daily-security-observability.lock.yml @@ -1305,12 +1305,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1337,6 +1338,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1367,7 +1372,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1435,12 +1440,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1503,6 +1511,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-security-red-team.lock.yml b/.github/workflows/daily-security-red-team.lock.yml index 2e92622855e..dc13f8006a2 100644 --- a/.github/workflows/daily-security-red-team.lock.yml +++ b/.github/workflows/daily-security-red-team.lock.yml @@ -1274,12 +1274,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1306,6 +1307,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1336,7 +1341,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1404,12 +1409,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1472,6 +1480,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-semgrep-scan.lock.yml b/.github/workflows/daily-semgrep-scan.lock.yml index 8cca628e685..4f5d135c764 100644 --- a/.github/workflows/daily-semgrep-scan.lock.yml +++ b/.github/workflows/daily-semgrep-scan.lock.yml @@ -1161,12 +1161,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1193,6 +1194,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1223,7 +1228,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1291,12 +1296,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1359,6 +1367,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-sentrux-report.lock.yml b/.github/workflows/daily-sentrux-report.lock.yml index ea97dddc3da..dadbbad84b3 100644 --- a/.github/workflows/daily-sentrux-report.lock.yml +++ b/.github/workflows/daily-sentrux-report.lock.yml @@ -1138,12 +1138,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1170,6 +1171,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1200,7 +1205,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1268,12 +1273,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1336,6 +1344,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-skill-optimizer.lock.yml b/.github/workflows/daily-skill-optimizer.lock.yml index 8d9bf922b77..197a0070441 100644 --- a/.github/workflows/daily-skill-optimizer.lock.yml +++ b/.github/workflows/daily-skill-optimizer.lock.yml @@ -1104,12 +1104,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1136,6 +1137,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1166,7 +1171,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1234,12 +1239,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1302,6 +1310,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-spdd-spec-planner.lock.yml b/.github/workflows/daily-spdd-spec-planner.lock.yml index e23e9e0b235..7c640a94b50 100644 --- a/.github/workflows/daily-spdd-spec-planner.lock.yml +++ b/.github/workflows/daily-spdd-spec-planner.lock.yml @@ -1166,12 +1166,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1198,6 +1199,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1228,7 +1233,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1296,12 +1301,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1364,6 +1372,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-syntax-error-quality.lock.yml b/.github/workflows/daily-syntax-error-quality.lock.yml index 5ef83fbd122..0afa3c72759 100644 --- a/.github/workflows/daily-syntax-error-quality.lock.yml +++ b/.github/workflows/daily-syntax-error-quality.lock.yml @@ -1105,12 +1105,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1137,6 +1138,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1167,7 +1172,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1235,12 +1240,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1303,6 +1311,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-team-evolution-insights.lock.yml b/.github/workflows/daily-team-evolution-insights.lock.yml index 06dda77df87..42c987f78bc 100644 --- a/.github/workflows/daily-team-evolution-insights.lock.yml +++ b/.github/workflows/daily-team-evolution-insights.lock.yml @@ -1146,12 +1146,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1178,6 +1179,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1208,7 +1213,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1276,12 +1281,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1344,6 +1352,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-team-status.lock.yml b/.github/workflows/daily-team-status.lock.yml index cc3d88947ef..8743f89fd0a 100644 --- a/.github/workflows/daily-team-status.lock.yml +++ b/.github/workflows/daily-team-status.lock.yml @@ -1062,12 +1062,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1094,6 +1095,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1124,7 +1129,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1192,12 +1197,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1260,6 +1268,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-testify-uber-super-expert.lock.yml b/.github/workflows/daily-testify-uber-super-expert.lock.yml index 4289a44ff19..82f0c3fb602 100644 --- a/.github/workflows/daily-testify-uber-super-expert.lock.yml +++ b/.github/workflows/daily-testify-uber-super-expert.lock.yml @@ -1210,12 +1210,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1242,6 +1243,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1272,7 +1277,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1340,12 +1345,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1408,6 +1416,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-token-consumption-report.lock.yml b/.github/workflows/daily-token-consumption-report.lock.yml index 35157234f55..0af77359882 100644 --- a/.github/workflows/daily-token-consumption-report.lock.yml +++ b/.github/workflows/daily-token-consumption-report.lock.yml @@ -1300,12 +1300,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1332,6 +1333,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1362,7 +1367,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1430,12 +1435,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1498,6 +1506,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-windows-terminal-integration-builder.lock.yml b/.github/workflows/daily-windows-terminal-integration-builder.lock.yml index 72186634285..40a18065ea4 100644 --- a/.github/workflows/daily-windows-terminal-integration-builder.lock.yml +++ b/.github/workflows/daily-windows-terminal-integration-builder.lock.yml @@ -1045,12 +1045,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1077,6 +1078,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1107,7 +1112,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1175,12 +1180,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1243,6 +1251,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/daily-workflow-updater.lock.yml b/.github/workflows/daily-workflow-updater.lock.yml index 8dd576b3f50..22cb0756692 100644 --- a/.github/workflows/daily-workflow-updater.lock.yml +++ b/.github/workflows/daily-workflow-updater.lock.yml @@ -1092,12 +1092,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1124,6 +1125,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1154,7 +1159,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1222,12 +1227,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1290,6 +1298,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/dataflow-pr-discussion-dataset.lock.yml b/.github/workflows/dataflow-pr-discussion-dataset.lock.yml index a38a7f6d458..582a8cc423b 100644 --- a/.github/workflows/dataflow-pr-discussion-dataset.lock.yml +++ b/.github/workflows/dataflow-pr-discussion-dataset.lock.yml @@ -1454,12 +1454,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1486,6 +1487,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1516,7 +1521,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1584,12 +1589,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1652,6 +1660,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/dead-code-remover.lock.yml b/.github/workflows/dead-code-remover.lock.yml index 609312d6185..6d6ca9ba5df 100644 --- a/.github/workflows/dead-code-remover.lock.yml +++ b/.github/workflows/dead-code-remover.lock.yml @@ -1164,12 +1164,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1196,6 +1197,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1226,7 +1231,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1294,12 +1299,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1362,6 +1370,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/deep-report.lock.yml b/.github/workflows/deep-report.lock.yml index bbb405d5b6d..fe0fc015317 100644 --- a/.github/workflows/deep-report.lock.yml +++ b/.github/workflows/deep-report.lock.yml @@ -1646,12 +1646,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1678,6 +1679,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1708,7 +1713,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1776,12 +1781,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1844,6 +1852,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/delight.lock.yml b/.github/workflows/delight.lock.yml index 97b4f4d1464..656b99509b1 100644 --- a/.github/workflows/delight.lock.yml +++ b/.github/workflows/delight.lock.yml @@ -1193,12 +1193,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1225,6 +1226,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1255,7 +1260,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1323,12 +1328,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1391,6 +1399,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/dependabot-burner.lock.yml b/.github/workflows/dependabot-burner.lock.yml index 95dfc93c8f3..e028b755f84 100644 --- a/.github/workflows/dependabot-burner.lock.yml +++ b/.github/workflows/dependabot-burner.lock.yml @@ -1236,12 +1236,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1268,6 +1269,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1298,7 +1303,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1366,12 +1371,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1434,6 +1442,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/dependabot-go-checker.lock.yml b/.github/workflows/dependabot-go-checker.lock.yml index 604378905de..9ca922af087 100644 --- a/.github/workflows/dependabot-go-checker.lock.yml +++ b/.github/workflows/dependabot-go-checker.lock.yml @@ -1152,12 +1152,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1184,6 +1185,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1214,7 +1219,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1282,12 +1287,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1350,6 +1358,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/dependabot-repair.lock.yml b/.github/workflows/dependabot-repair.lock.yml index 3a98a8a038a..d286b8e9d0a 100644 --- a/.github/workflows/dependabot-repair.lock.yml +++ b/.github/workflows/dependabot-repair.lock.yml @@ -1192,12 +1192,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1224,6 +1225,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1254,7 +1259,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1322,12 +1327,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1390,6 +1398,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/deployment-incident-monitor.lock.yml b/.github/workflows/deployment-incident-monitor.lock.yml index abc76efff80..49d52b89571 100644 --- a/.github/workflows/deployment-incident-monitor.lock.yml +++ b/.github/workflows/deployment-incident-monitor.lock.yml @@ -1102,12 +1102,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1134,6 +1135,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1164,7 +1169,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1232,12 +1237,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1300,6 +1308,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/design-decision-gate.lock.yml b/.github/workflows/design-decision-gate.lock.yml index f67657911bf..eb68b658c88 100644 --- a/.github/workflows/design-decision-gate.lock.yml +++ b/.github/workflows/design-decision-gate.lock.yml @@ -1287,12 +1287,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1319,6 +1320,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1349,7 +1354,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1417,12 +1422,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1485,6 +1493,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/designer-drift-audit.lock.yml b/.github/workflows/designer-drift-audit.lock.yml index 61e8464ff6c..327f7945190 100644 --- a/.github/workflows/designer-drift-audit.lock.yml +++ b/.github/workflows/designer-drift-audit.lock.yml @@ -1051,12 +1051,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1083,6 +1084,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1113,7 +1118,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1181,12 +1186,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1249,6 +1257,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/dev-hawk.lock.yml b/.github/workflows/dev-hawk.lock.yml index 6e4ea77eb49..039bfdd4467 100644 --- a/.github/workflows/dev-hawk.lock.yml +++ b/.github/workflows/dev-hawk.lock.yml @@ -1210,12 +1210,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1242,6 +1243,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1272,7 +1277,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1340,12 +1345,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1408,6 +1416,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/dev.lock.yml b/.github/workflows/dev.lock.yml index cf732f9fb54..023cb6f515f 100644 --- a/.github/workflows/dev.lock.yml +++ b/.github/workflows/dev.lock.yml @@ -1170,12 +1170,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1202,6 +1203,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1232,7 +1237,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1300,12 +1305,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1368,6 +1376,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/developer-docs-consolidator.lock.yml b/.github/workflows/developer-docs-consolidator.lock.yml index 8460f519437..545dd5b2d1e 100644 --- a/.github/workflows/developer-docs-consolidator.lock.yml +++ b/.github/workflows/developer-docs-consolidator.lock.yml @@ -1343,12 +1343,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1375,6 +1376,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1405,7 +1410,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1473,12 +1478,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1541,6 +1549,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/dictation-prompt.lock.yml b/.github/workflows/dictation-prompt.lock.yml index bbdea75520b..673cf77827c 100644 --- a/.github/workflows/dictation-prompt.lock.yml +++ b/.github/workflows/dictation-prompt.lock.yml @@ -1094,12 +1094,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1126,6 +1127,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1156,7 +1161,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1224,12 +1229,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1292,6 +1300,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/discussion-task-miner.lock.yml b/.github/workflows/discussion-task-miner.lock.yml index bdcbda2efa5..b4f0710ed98 100644 --- a/.github/workflows/discussion-task-miner.lock.yml +++ b/.github/workflows/discussion-task-miner.lock.yml @@ -1176,12 +1176,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1208,6 +1209,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1238,7 +1243,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1306,12 +1311,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1374,6 +1382,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/docs-noob-tester.lock.yml b/.github/workflows/docs-noob-tester.lock.yml index 800f7c73f3a..2a195b445f4 100644 --- a/.github/workflows/docs-noob-tester.lock.yml +++ b/.github/workflows/docs-noob-tester.lock.yml @@ -1146,12 +1146,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1178,6 +1179,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1208,7 +1213,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1276,12 +1281,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1344,6 +1352,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/draft-pr-cleanup.lock.yml b/.github/workflows/draft-pr-cleanup.lock.yml index d78ce421fe8..7531a4327ba 100644 --- a/.github/workflows/draft-pr-cleanup.lock.yml +++ b/.github/workflows/draft-pr-cleanup.lock.yml @@ -1128,12 +1128,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1160,6 +1161,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1190,7 +1195,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1258,12 +1263,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1326,6 +1334,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/duplicate-code-detector.lock.yml b/.github/workflows/duplicate-code-detector.lock.yml index 6312ea4f36d..3b07fe06f73 100644 --- a/.github/workflows/duplicate-code-detector.lock.yml +++ b/.github/workflows/duplicate-code-detector.lock.yml @@ -1187,12 +1187,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1219,6 +1220,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1249,7 +1254,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1317,12 +1322,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1385,6 +1393,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/example-failure-category-filter.lock.yml b/.github/workflows/example-failure-category-filter.lock.yml index de6a5c04e39..8983ead27b9 100644 --- a/.github/workflows/example-failure-category-filter.lock.yml +++ b/.github/workflows/example-failure-category-filter.lock.yml @@ -1039,12 +1039,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1071,6 +1072,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1101,7 +1106,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1169,12 +1174,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1237,6 +1245,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/example-permissions-warning.lock.yml b/.github/workflows/example-permissions-warning.lock.yml index e5df06d4aec..7e3f040dd5f 100644 --- a/.github/workflows/example-permissions-warning.lock.yml +++ b/.github/workflows/example-permissions-warning.lock.yml @@ -1002,12 +1002,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1034,6 +1035,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1064,7 +1069,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1132,12 +1137,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1200,6 +1208,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/example-workflow-analyzer.lock.yml b/.github/workflows/example-workflow-analyzer.lock.yml index 57b5f71813d..f4c1aeff624 100644 --- a/.github/workflows/example-workflow-analyzer.lock.yml +++ b/.github/workflows/example-workflow-analyzer.lock.yml @@ -1226,12 +1226,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1258,6 +1259,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1288,7 +1293,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1356,12 +1361,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1424,6 +1432,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/firewall-escape.lock.yml b/.github/workflows/firewall-escape.lock.yml index f6ed58eed67..2f01e1f7548 100644 --- a/.github/workflows/firewall-escape.lock.yml +++ b/.github/workflows/firewall-escape.lock.yml @@ -1187,12 +1187,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1219,6 +1220,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1249,7 +1254,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1317,12 +1322,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1385,6 +1393,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/firewall.lock.yml b/.github/workflows/firewall.lock.yml index 1d11dc16e64..c324621094c 100644 --- a/.github/workflows/firewall.lock.yml +++ b/.github/workflows/firewall.lock.yml @@ -1010,12 +1010,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1042,6 +1043,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1072,7 +1077,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1140,12 +1145,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1208,6 +1216,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/functional-pragmatist.lock.yml b/.github/workflows/functional-pragmatist.lock.yml index 909c083b5e0..56175a98bb0 100644 --- a/.github/workflows/functional-pragmatist.lock.yml +++ b/.github/workflows/functional-pragmatist.lock.yml @@ -1100,12 +1100,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1132,6 +1133,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1162,7 +1167,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1230,12 +1235,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1298,6 +1306,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/github-mcp-structural-analysis.lock.yml b/.github/workflows/github-mcp-structural-analysis.lock.yml index 9341acae493..fb168673a00 100644 --- a/.github/workflows/github-mcp-structural-analysis.lock.yml +++ b/.github/workflows/github-mcp-structural-analysis.lock.yml @@ -1249,12 +1249,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1281,6 +1282,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1311,7 +1316,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1379,12 +1384,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1447,6 +1455,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/github-mcp-tools-report.lock.yml b/.github/workflows/github-mcp-tools-report.lock.yml index 9ef76228e2b..aebaaa82620 100644 --- a/.github/workflows/github-mcp-tools-report.lock.yml +++ b/.github/workflows/github-mcp-tools-report.lock.yml @@ -1240,12 +1240,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1272,6 +1273,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1302,7 +1307,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1370,12 +1375,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1438,6 +1446,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/github-remote-mcp-auth-test.lock.yml b/.github/workflows/github-remote-mcp-auth-test.lock.yml index a5e48996683..7c04eb1f8bc 100644 --- a/.github/workflows/github-remote-mcp-auth-test.lock.yml +++ b/.github/workflows/github-remote-mcp-auth-test.lock.yml @@ -1096,12 +1096,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1128,6 +1129,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1158,7 +1163,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1226,12 +1231,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1294,6 +1302,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/glossary-maintainer.lock.yml b/.github/workflows/glossary-maintainer.lock.yml index 9ac3629e54f..9704caf9711 100644 --- a/.github/workflows/glossary-maintainer.lock.yml +++ b/.github/workflows/glossary-maintainer.lock.yml @@ -1244,12 +1244,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1276,6 +1277,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1306,7 +1311,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1374,12 +1379,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1442,6 +1450,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/go-fan.lock.yml b/.github/workflows/go-fan.lock.yml index e12c9837dfc..c0b1f0ce098 100644 --- a/.github/workflows/go-fan.lock.yml +++ b/.github/workflows/go-fan.lock.yml @@ -1272,12 +1272,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1304,6 +1305,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1334,7 +1339,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1402,12 +1407,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1470,6 +1478,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/go-logger.lock.yml b/.github/workflows/go-logger.lock.yml index cf48dd76012..b97f9db4999 100644 --- a/.github/workflows/go-logger.lock.yml +++ b/.github/workflows/go-logger.lock.yml @@ -1256,12 +1256,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1288,6 +1289,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1318,7 +1323,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1386,12 +1391,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1454,6 +1462,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/go-pattern-detector.lock.yml b/.github/workflows/go-pattern-detector.lock.yml index 371f3dfee7d..2b90f7d10d5 100644 --- a/.github/workflows/go-pattern-detector.lock.yml +++ b/.github/workflows/go-pattern-detector.lock.yml @@ -1220,12 +1220,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1252,6 +1253,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1282,7 +1287,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1350,12 +1355,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1418,6 +1426,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/gpclean.lock.yml b/.github/workflows/gpclean.lock.yml index b9da4a893ee..f66b7462078 100644 --- a/.github/workflows/gpclean.lock.yml +++ b/.github/workflows/gpclean.lock.yml @@ -1181,12 +1181,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1213,6 +1214,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1243,7 +1248,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1311,12 +1316,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1379,6 +1387,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/grumpy-reviewer.lock.yml b/.github/workflows/grumpy-reviewer.lock.yml index 05bc6475cd2..d4cdee114d6 100644 --- a/.github/workflows/grumpy-reviewer.lock.yml +++ b/.github/workflows/grumpy-reviewer.lock.yml @@ -1222,12 +1222,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1254,6 +1255,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1284,7 +1289,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1352,12 +1357,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1420,6 +1428,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/hippo-embed.lock.yml b/.github/workflows/hippo-embed.lock.yml index 96e6f0565d3..55f27fc54db 100644 --- a/.github/workflows/hippo-embed.lock.yml +++ b/.github/workflows/hippo-embed.lock.yml @@ -1132,12 +1132,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1164,6 +1165,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1194,7 +1199,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1262,12 +1267,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1330,6 +1338,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/hourly-ci-cleaner.lock.yml b/.github/workflows/hourly-ci-cleaner.lock.yml index cabba4b2f65..8bc2573c0a6 100644 --- a/.github/workflows/hourly-ci-cleaner.lock.yml +++ b/.github/workflows/hourly-ci-cleaner.lock.yml @@ -1257,12 +1257,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1289,6 +1290,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1319,7 +1324,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1387,12 +1392,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1455,6 +1463,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/instructions-janitor.lock.yml b/.github/workflows/instructions-janitor.lock.yml index c7d52e49b15..e9a9e5869e7 100644 --- a/.github/workflows/instructions-janitor.lock.yml +++ b/.github/workflows/instructions-janitor.lock.yml @@ -1231,12 +1231,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1263,6 +1264,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1293,7 +1298,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1361,12 +1366,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1429,6 +1437,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/issue-arborist.lock.yml b/.github/workflows/issue-arborist.lock.yml index fc92dafdc22..ef5571a9c4f 100644 --- a/.github/workflows/issue-arborist.lock.yml +++ b/.github/workflows/issue-arborist.lock.yml @@ -1253,12 +1253,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1285,6 +1286,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1315,7 +1320,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1383,12 +1388,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1451,6 +1459,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/issue-monster.lock.yml b/.github/workflows/issue-monster.lock.yml index 6d0f5c08c76..5d96fe414e2 100644 --- a/.github/workflows/issue-monster.lock.yml +++ b/.github/workflows/issue-monster.lock.yml @@ -1470,12 +1470,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1502,6 +1503,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1532,7 +1537,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1600,12 +1605,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1668,6 +1676,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/issue-triage-agent.lock.yml b/.github/workflows/issue-triage-agent.lock.yml index 10526a37be3..9e1af2c683c 100644 --- a/.github/workflows/issue-triage-agent.lock.yml +++ b/.github/workflows/issue-triage-agent.lock.yml @@ -1076,12 +1076,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1108,6 +1109,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1138,7 +1143,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1206,12 +1211,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1274,6 +1282,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/jsweep.lock.yml b/.github/workflows/jsweep.lock.yml index c70c90f198f..c4bd8c3b290 100644 --- a/.github/workflows/jsweep.lock.yml +++ b/.github/workflows/jsweep.lock.yml @@ -1152,12 +1152,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1184,6 +1185,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1214,7 +1219,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1282,12 +1287,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1350,6 +1358,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/layout-spec-maintainer.lock.yml b/.github/workflows/layout-spec-maintainer.lock.yml index 80646f0f724..894d4b2ff57 100644 --- a/.github/workflows/layout-spec-maintainer.lock.yml +++ b/.github/workflows/layout-spec-maintainer.lock.yml @@ -1140,12 +1140,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1172,6 +1173,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1202,7 +1207,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1270,12 +1275,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1338,6 +1346,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/lint-monster.lock.yml b/.github/workflows/lint-monster.lock.yml index 7f14018503f..03265a7fadd 100644 --- a/.github/workflows/lint-monster.lock.yml +++ b/.github/workflows/lint-monster.lock.yml @@ -1185,12 +1185,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1217,6 +1218,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1247,7 +1252,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1315,12 +1320,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1383,6 +1391,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/linter-miner.lock.yml b/.github/workflows/linter-miner.lock.yml index 5a64922e9d2..f49b5b0e3e0 100644 --- a/.github/workflows/linter-miner.lock.yml +++ b/.github/workflows/linter-miner.lock.yml @@ -1182,12 +1182,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1214,6 +1215,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1244,7 +1249,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1312,12 +1317,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1380,6 +1388,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/lockfile-stats.lock.yml b/.github/workflows/lockfile-stats.lock.yml index 6363505b941..570d9a17896 100644 --- a/.github/workflows/lockfile-stats.lock.yml +++ b/.github/workflows/lockfile-stats.lock.yml @@ -1191,12 +1191,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1223,6 +1224,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1253,7 +1258,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1321,12 +1326,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1389,6 +1397,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/mattpocock-skills-reviewer.lock.yml b/.github/workflows/mattpocock-skills-reviewer.lock.yml index 7aa848ebc7b..2ce6cc911c1 100644 --- a/.github/workflows/mattpocock-skills-reviewer.lock.yml +++ b/.github/workflows/mattpocock-skills-reviewer.lock.yml @@ -1218,12 +1218,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1250,6 +1251,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1280,7 +1285,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1348,12 +1353,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1416,6 +1424,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/mcp-inspector.lock.yml b/.github/workflows/mcp-inspector.lock.yml index d49cb136c25..c84d7af875f 100644 --- a/.github/workflows/mcp-inspector.lock.yml +++ b/.github/workflows/mcp-inspector.lock.yml @@ -1661,12 +1661,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1693,6 +1694,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1723,7 +1728,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1791,12 +1796,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1859,6 +1867,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/mergefest.lock.yml b/.github/workflows/mergefest.lock.yml index 0c5e2628e36..0a0f4a27760 100644 --- a/.github/workflows/mergefest.lock.yml +++ b/.github/workflows/mergefest.lock.yml @@ -1179,12 +1179,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1211,6 +1212,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1241,7 +1246,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1309,12 +1314,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1377,6 +1385,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/metrics-collector.lock.yml b/.github/workflows/metrics-collector.lock.yml index e893dd6b91d..cfd0b3484d7 100644 --- a/.github/workflows/metrics-collector.lock.yml +++ b/.github/workflows/metrics-collector.lock.yml @@ -1221,12 +1221,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1253,6 +1254,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1283,7 +1288,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1351,12 +1356,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1419,6 +1427,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/necromancer.lock.yml b/.github/workflows/necromancer.lock.yml index 0200e3e383a..0b81d927a76 100644 --- a/.github/workflows/necromancer.lock.yml +++ b/.github/workflows/necromancer.lock.yml @@ -1197,12 +1197,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1229,6 +1230,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1259,7 +1264,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1327,12 +1332,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1395,6 +1403,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/notion-issue-summary.lock.yml b/.github/workflows/notion-issue-summary.lock.yml index a9cf6f9dcab..7bf317ac1e5 100644 --- a/.github/workflows/notion-issue-summary.lock.yml +++ b/.github/workflows/notion-issue-summary.lock.yml @@ -1093,12 +1093,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1125,6 +1126,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1155,7 +1160,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1223,12 +1228,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1291,6 +1299,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/objective-impact-report.lock.yml b/.github/workflows/objective-impact-report.lock.yml index 9bb8b3f31af..585ae1267bb 100644 --- a/.github/workflows/objective-impact-report.lock.yml +++ b/.github/workflows/objective-impact-report.lock.yml @@ -1098,12 +1098,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1130,6 +1131,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1160,7 +1165,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1228,12 +1233,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1296,6 +1304,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/org-health-report.lock.yml b/.github/workflows/org-health-report.lock.yml index 825be0435f6..f47ce25bcfd 100644 --- a/.github/workflows/org-health-report.lock.yml +++ b/.github/workflows/org-health-report.lock.yml @@ -1194,12 +1194,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1226,6 +1227,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1256,7 +1261,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1324,12 +1329,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1392,6 +1400,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/outcome-collector.lock.yml b/.github/workflows/outcome-collector.lock.yml index 52c6d9c2225..c5100e1117f 100644 --- a/.github/workflows/outcome-collector.lock.yml +++ b/.github/workflows/outcome-collector.lock.yml @@ -1141,12 +1141,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1173,6 +1174,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1203,7 +1208,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1271,12 +1276,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1339,6 +1347,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/pdf-summary.lock.yml b/.github/workflows/pdf-summary.lock.yml index e00a929d180..b547b667226 100644 --- a/.github/workflows/pdf-summary.lock.yml +++ b/.github/workflows/pdf-summary.lock.yml @@ -1253,12 +1253,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1285,6 +1286,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1315,7 +1320,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1383,12 +1388,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1451,6 +1459,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/plan.lock.yml b/.github/workflows/plan.lock.yml index f26c7b897c9..af03edd15d9 100644 --- a/.github/workflows/plan.lock.yml +++ b/.github/workflows/plan.lock.yml @@ -1182,12 +1182,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1214,6 +1215,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1244,7 +1249,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1312,12 +1317,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1380,6 +1388,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/poem-bot.lock.yml b/.github/workflows/poem-bot.lock.yml index 0806bf67763..f0a2a71acf4 100644 --- a/.github/workflows/poem-bot.lock.yml +++ b/.github/workflows/poem-bot.lock.yml @@ -1464,12 +1464,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1496,6 +1497,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1526,7 +1531,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1594,12 +1599,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1662,6 +1670,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/portfolio-analyst.lock.yml b/.github/workflows/portfolio-analyst.lock.yml index 8269113af8d..76f5854b758 100644 --- a/.github/workflows/portfolio-analyst.lock.yml +++ b/.github/workflows/portfolio-analyst.lock.yml @@ -1320,12 +1320,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1352,6 +1353,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1382,7 +1387,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1450,12 +1455,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1518,6 +1526,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/pr-code-quality-reviewer.lock.yml b/.github/workflows/pr-code-quality-reviewer.lock.yml index 9e70da71fdf..12e5d5a63a6 100644 --- a/.github/workflows/pr-code-quality-reviewer.lock.yml +++ b/.github/workflows/pr-code-quality-reviewer.lock.yml @@ -1178,12 +1178,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1210,6 +1211,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1240,7 +1245,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1308,12 +1313,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1376,6 +1384,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/pr-description-caveman.lock.yml b/.github/workflows/pr-description-caveman.lock.yml index 08df8ff6779..4811ba3de56 100644 --- a/.github/workflows/pr-description-caveman.lock.yml +++ b/.github/workflows/pr-description-caveman.lock.yml @@ -1100,12 +1100,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1132,6 +1133,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1162,7 +1167,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1230,12 +1235,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1298,6 +1306,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/pr-nitpick-reviewer.lock.yml b/.github/workflows/pr-nitpick-reviewer.lock.yml index 7406d65c71e..168e770142d 100644 --- a/.github/workflows/pr-nitpick-reviewer.lock.yml +++ b/.github/workflows/pr-nitpick-reviewer.lock.yml @@ -1222,12 +1222,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1254,6 +1255,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1284,7 +1289,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1352,12 +1357,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1420,6 +1428,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/pr-sous-chef.lock.yml b/.github/workflows/pr-sous-chef.lock.yml index a55004f0818..bedac56ecd5 100644 --- a/.github/workflows/pr-sous-chef.lock.yml +++ b/.github/workflows/pr-sous-chef.lock.yml @@ -1220,12 +1220,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1252,6 +1253,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1282,7 +1287,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1350,12 +1355,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1418,6 +1426,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/pr-triage-agent.lock.yml b/.github/workflows/pr-triage-agent.lock.yml index e544e172448..e250673abae 100644 --- a/.github/workflows/pr-triage-agent.lock.yml +++ b/.github/workflows/pr-triage-agent.lock.yml @@ -1246,12 +1246,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1278,6 +1279,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1308,7 +1313,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1376,12 +1381,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1444,6 +1452,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/prompt-clustering-analysis.lock.yml b/.github/workflows/prompt-clustering-analysis.lock.yml index 31c0ec916c1..586b886049f 100644 --- a/.github/workflows/prompt-clustering-analysis.lock.yml +++ b/.github/workflows/prompt-clustering-analysis.lock.yml @@ -1362,12 +1362,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1394,6 +1395,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1424,7 +1429,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1492,12 +1497,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1560,6 +1568,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/python-data-charts.lock.yml b/.github/workflows/python-data-charts.lock.yml index eefe8207f91..68f1a1a56ac 100644 --- a/.github/workflows/python-data-charts.lock.yml +++ b/.github/workflows/python-data-charts.lock.yml @@ -1277,12 +1277,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1309,6 +1310,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1339,7 +1344,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1407,12 +1412,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1475,6 +1483,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/q.lock.yml b/.github/workflows/q.lock.yml index 8a8fd934fab..a298d7d14e4 100644 --- a/.github/workflows/q.lock.yml +++ b/.github/workflows/q.lock.yml @@ -1324,12 +1324,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1356,6 +1357,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1386,7 +1391,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1454,12 +1459,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1522,6 +1530,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/refactoring-cadence.lock.yml b/.github/workflows/refactoring-cadence.lock.yml index 45218804761..cc8e67c01cc 100644 --- a/.github/workflows/refactoring-cadence.lock.yml +++ b/.github/workflows/refactoring-cadence.lock.yml @@ -1134,12 +1134,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1166,6 +1167,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1196,7 +1201,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1264,12 +1269,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1332,6 +1340,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/refiner.lock.yml b/.github/workflows/refiner.lock.yml index 6ac397606a3..151d0327f6d 100644 --- a/.github/workflows/refiner.lock.yml +++ b/.github/workflows/refiner.lock.yml @@ -1222,12 +1222,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1254,6 +1255,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1284,7 +1289,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1352,12 +1357,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1420,6 +1428,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/release.lock.yml b/.github/workflows/release.lock.yml index a6c850dab74..6886c10a918 100644 --- a/.github/workflows/release.lock.yml +++ b/.github/workflows/release.lock.yml @@ -1136,12 +1136,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1168,6 +1169,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1198,7 +1203,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1266,12 +1271,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1334,6 +1342,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/repo-audit-analyzer.lock.yml b/.github/workflows/repo-audit-analyzer.lock.yml index fa8a92fc713..841559fa8fb 100644 --- a/.github/workflows/repo-audit-analyzer.lock.yml +++ b/.github/workflows/repo-audit-analyzer.lock.yml @@ -1132,12 +1132,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1164,6 +1165,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1194,7 +1199,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1262,12 +1267,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1330,6 +1338,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/repo-tree-map.lock.yml b/.github/workflows/repo-tree-map.lock.yml index 22a9e6dffed..34c0f8389f7 100644 --- a/.github/workflows/repo-tree-map.lock.yml +++ b/.github/workflows/repo-tree-map.lock.yml @@ -1081,12 +1081,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1113,6 +1114,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1143,7 +1148,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1211,12 +1216,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1279,6 +1287,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/repository-quality-improver.lock.yml b/.github/workflows/repository-quality-improver.lock.yml index a0c2a1e837c..106fbe4e308 100644 --- a/.github/workflows/repository-quality-improver.lock.yml +++ b/.github/workflows/repository-quality-improver.lock.yml @@ -1135,12 +1135,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1167,6 +1168,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1197,7 +1202,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1265,12 +1270,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1333,6 +1341,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/research.lock.yml b/.github/workflows/research.lock.yml index 3605dcfa970..d7cfca71dfa 100644 --- a/.github/workflows/research.lock.yml +++ b/.github/workflows/research.lock.yml @@ -1111,12 +1111,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1143,6 +1144,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1173,7 +1178,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1241,12 +1246,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1309,6 +1317,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/ruflo-backed-task.lock.yml b/.github/workflows/ruflo-backed-task.lock.yml index efb6a3c8e67..39a226a9f3d 100644 --- a/.github/workflows/ruflo-backed-task.lock.yml +++ b/.github/workflows/ruflo-backed-task.lock.yml @@ -1290,12 +1290,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1322,6 +1323,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1352,7 +1357,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1420,12 +1425,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1488,6 +1496,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/safe-output-health.lock.yml b/.github/workflows/safe-output-health.lock.yml index ebd77df9bb6..e2f39a3bdc5 100644 --- a/.github/workflows/safe-output-health.lock.yml +++ b/.github/workflows/safe-output-health.lock.yml @@ -1307,12 +1307,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1339,6 +1340,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1369,7 +1374,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1437,12 +1442,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1505,6 +1513,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/schema-consistency-checker.lock.yml b/.github/workflows/schema-consistency-checker.lock.yml index 96c44f4ba37..ba78eafa036 100644 --- a/.github/workflows/schema-consistency-checker.lock.yml +++ b/.github/workflows/schema-consistency-checker.lock.yml @@ -1108,12 +1108,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1140,6 +1141,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1170,7 +1175,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1238,12 +1243,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1306,6 +1314,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/schema-feature-coverage.lock.yml b/.github/workflows/schema-feature-coverage.lock.yml index 0ec75057950..5501f464df2 100644 --- a/.github/workflows/schema-feature-coverage.lock.yml +++ b/.github/workflows/schema-feature-coverage.lock.yml @@ -1152,12 +1152,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1184,6 +1185,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1214,7 +1219,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1282,12 +1287,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1350,6 +1358,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/scout.lock.yml b/.github/workflows/scout.lock.yml index 5a855c3e1ca..f4dab02b99b 100644 --- a/.github/workflows/scout.lock.yml +++ b/.github/workflows/scout.lock.yml @@ -1389,12 +1389,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1421,6 +1422,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1451,7 +1456,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1519,12 +1524,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1587,6 +1595,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/security-compliance.lock.yml b/.github/workflows/security-compliance.lock.yml index 2d31b0ab548..d84fb686829 100644 --- a/.github/workflows/security-compliance.lock.yml +++ b/.github/workflows/security-compliance.lock.yml @@ -1140,12 +1140,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1172,6 +1173,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1202,7 +1207,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1270,12 +1275,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1338,6 +1346,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/security-review.lock.yml b/.github/workflows/security-review.lock.yml index 603a1ebe61e..adcacbffa60 100644 --- a/.github/workflows/security-review.lock.yml +++ b/.github/workflows/security-review.lock.yml @@ -1266,12 +1266,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1298,6 +1299,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1328,7 +1333,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1396,12 +1401,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1464,6 +1472,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/semantic-function-refactor.lock.yml b/.github/workflows/semantic-function-refactor.lock.yml index 61d7cb8f865..93c9a38f9e1 100644 --- a/.github/workflows/semantic-function-refactor.lock.yml +++ b/.github/workflows/semantic-function-refactor.lock.yml @@ -1226,12 +1226,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1258,6 +1259,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1288,7 +1293,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1356,12 +1361,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1424,6 +1432,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/sergo.lock.yml b/.github/workflows/sergo.lock.yml index 9bb7dad8ee6..a967a821f06 100644 --- a/.github/workflows/sergo.lock.yml +++ b/.github/workflows/sergo.lock.yml @@ -1276,12 +1276,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1308,6 +1309,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1338,7 +1343,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1406,12 +1411,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1474,6 +1482,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/skillet.lock.yml b/.github/workflows/skillet.lock.yml index c9de8b0d749..16767f60b97 100644 --- a/.github/workflows/skillet.lock.yml +++ b/.github/workflows/skillet.lock.yml @@ -1192,12 +1192,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1224,6 +1225,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1254,7 +1259,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1322,12 +1327,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1390,6 +1398,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/slide-deck-maintainer.lock.yml b/.github/workflows/slide-deck-maintainer.lock.yml index 15f58709dd9..bfdd1543892 100644 --- a/.github/workflows/slide-deck-maintainer.lock.yml +++ b/.github/workflows/slide-deck-maintainer.lock.yml @@ -1232,12 +1232,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1264,6 +1265,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1294,7 +1299,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1362,12 +1367,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1430,6 +1438,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-agent-all-merged.lock.yml b/.github/workflows/smoke-agent-all-merged.lock.yml index 4b1a20c74b3..dfa90651bb0 100644 --- a/.github/workflows/smoke-agent-all-merged.lock.yml +++ b/.github/workflows/smoke-agent-all-merged.lock.yml @@ -1192,12 +1192,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1224,6 +1225,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1254,7 +1259,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1322,12 +1327,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1390,6 +1398,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-agent-all-none.lock.yml b/.github/workflows/smoke-agent-all-none.lock.yml index c605bbfae97..bf36328f12a 100644 --- a/.github/workflows/smoke-agent-all-none.lock.yml +++ b/.github/workflows/smoke-agent-all-none.lock.yml @@ -1192,12 +1192,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1224,6 +1225,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1254,7 +1259,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1322,12 +1327,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1390,6 +1398,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-agent-public-approved.lock.yml b/.github/workflows/smoke-agent-public-approved.lock.yml index 4c16921a511..8e7f523bf1c 100644 --- a/.github/workflows/smoke-agent-public-approved.lock.yml +++ b/.github/workflows/smoke-agent-public-approved.lock.yml @@ -1223,12 +1223,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1255,6 +1256,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1285,7 +1290,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1353,12 +1358,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1421,6 +1429,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-agent-public-none.lock.yml b/.github/workflows/smoke-agent-public-none.lock.yml index 378b2e56350..ea7af2e5a43 100644 --- a/.github/workflows/smoke-agent-public-none.lock.yml +++ b/.github/workflows/smoke-agent-public-none.lock.yml @@ -1192,12 +1192,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1224,6 +1225,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1254,7 +1259,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1322,12 +1327,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1390,6 +1398,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-agent-scoped-approved.lock.yml b/.github/workflows/smoke-agent-scoped-approved.lock.yml index 59b024cc41c..d83947fb82e 100644 --- a/.github/workflows/smoke-agent-scoped-approved.lock.yml +++ b/.github/workflows/smoke-agent-scoped-approved.lock.yml @@ -1199,12 +1199,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1231,6 +1232,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1261,7 +1266,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1329,12 +1334,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1397,6 +1405,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-antigravity.lock.yml b/.github/workflows/smoke-antigravity.lock.yml index 56ae9323cac..d44ab7e8c79 100644 --- a/.github/workflows/smoke-antigravity.lock.yml +++ b/.github/workflows/smoke-antigravity.lock.yml @@ -1260,12 +1260,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1292,6 +1293,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1322,7 +1327,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1390,12 +1395,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1458,6 +1466,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-call-workflow.lock.yml b/.github/workflows/smoke-call-workflow.lock.yml index d7b7e25681d..9bcfd88cc40 100644 --- a/.github/workflows/smoke-call-workflow.lock.yml +++ b/.github/workflows/smoke-call-workflow.lock.yml @@ -1194,12 +1194,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1226,6 +1227,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1256,7 +1261,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1324,12 +1329,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1392,6 +1400,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-ci.lock.yml b/.github/workflows/smoke-ci.lock.yml index a7f24626dd8..c19aa4b9845 100644 --- a/.github/workflows/smoke-ci.lock.yml +++ b/.github/workflows/smoke-ci.lock.yml @@ -1383,12 +1383,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1415,6 +1416,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1445,7 +1450,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1513,12 +1518,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1581,6 +1589,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-claude.lock.yml b/.github/workflows/smoke-claude.lock.yml index c9c084e73ab..59813476617 100644 --- a/.github/workflows/smoke-claude.lock.yml +++ b/.github/workflows/smoke-claude.lock.yml @@ -2027,12 +2027,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -2059,6 +2060,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -2089,7 +2094,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -2157,12 +2162,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -2225,6 +2233,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-codex.lock.yml b/.github/workflows/smoke-codex.lock.yml index bae7f03b419..3f5639742f8 100644 --- a/.github/workflows/smoke-codex.lock.yml +++ b/.github/workflows/smoke-codex.lock.yml @@ -1555,12 +1555,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1587,6 +1588,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1617,7 +1622,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1685,12 +1690,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1753,6 +1761,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-copilot-aoai-apikey.lock.yml b/.github/workflows/smoke-copilot-aoai-apikey.lock.yml index dd972b12420..059de7c4658 100644 --- a/.github/workflows/smoke-copilot-aoai-apikey.lock.yml +++ b/.github/workflows/smoke-copilot-aoai-apikey.lock.yml @@ -2195,12 +2195,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -2227,6 +2228,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -2257,7 +2262,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -2325,12 +2330,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -2393,6 +2401,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-copilot-aoai-entra.lock.yml b/.github/workflows/smoke-copilot-aoai-entra.lock.yml index 8372c79cd7a..0bd4fdeb442 100644 --- a/.github/workflows/smoke-copilot-aoai-entra.lock.yml +++ b/.github/workflows/smoke-copilot-aoai-entra.lock.yml @@ -2199,12 +2199,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -2231,6 +2232,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -2261,7 +2266,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -2329,12 +2334,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -2397,6 +2405,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-copilot-arm.lock.yml b/.github/workflows/smoke-copilot-arm.lock.yml index 25b0aa966a0..bf9fafcc608 100644 --- a/.github/workflows/smoke-copilot-arm.lock.yml +++ b/.github/workflows/smoke-copilot-arm.lock.yml @@ -2053,12 +2053,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -2085,6 +2086,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -2115,7 +2120,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -2183,12 +2188,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -2251,6 +2259,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-copilot-sdk.lock.yml b/.github/workflows/smoke-copilot-sdk.lock.yml index c0971db31c8..cfeebcaf51b 100644 --- a/.github/workflows/smoke-copilot-sdk.lock.yml +++ b/.github/workflows/smoke-copilot-sdk.lock.yml @@ -1129,12 +1129,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1161,6 +1162,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1191,7 +1196,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1259,12 +1264,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1327,6 +1335,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-copilot.lock.yml b/.github/workflows/smoke-copilot.lock.yml index 9dcff49fc9e..ab64744cbdb 100644 --- a/.github/workflows/smoke-copilot.lock.yml +++ b/.github/workflows/smoke-copilot.lock.yml @@ -2197,12 +2197,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -2229,6 +2230,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -2259,7 +2264,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -2327,12 +2332,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -2395,6 +2403,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-create-cross-repo-pr.lock.yml b/.github/workflows/smoke-create-cross-repo-pr.lock.yml index 727dd97492e..32fa92ff091 100644 --- a/.github/workflows/smoke-create-cross-repo-pr.lock.yml +++ b/.github/workflows/smoke-create-cross-repo-pr.lock.yml @@ -1259,12 +1259,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1291,6 +1292,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1321,7 +1326,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1389,12 +1394,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1457,6 +1465,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-crush.lock.yml b/.github/workflows/smoke-crush.lock.yml index 967a7df7cde..3c71ed77c84 100644 --- a/.github/workflows/smoke-crush.lock.yml +++ b/.github/workflows/smoke-crush.lock.yml @@ -1158,12 +1158,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1190,6 +1191,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1220,7 +1225,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1288,12 +1293,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1356,6 +1364,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-gemini.lock.yml b/.github/workflows/smoke-gemini.lock.yml index 894b315e2fa..a8ab6adf95e 100644 --- a/.github/workflows/smoke-gemini.lock.yml +++ b/.github/workflows/smoke-gemini.lock.yml @@ -1263,12 +1263,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1295,6 +1296,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1325,7 +1330,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1393,12 +1398,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1461,6 +1469,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-multi-pr.lock.yml b/.github/workflows/smoke-multi-pr.lock.yml index dfadf3f3f9d..01c900a6cf5 100644 --- a/.github/workflows/smoke-multi-pr.lock.yml +++ b/.github/workflows/smoke-multi-pr.lock.yml @@ -1204,12 +1204,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1236,6 +1237,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1266,7 +1271,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1334,12 +1339,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1402,6 +1410,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-opencode.lock.yml b/.github/workflows/smoke-opencode.lock.yml index f9bfd96e524..d62bd4513c8 100644 --- a/.github/workflows/smoke-opencode.lock.yml +++ b/.github/workflows/smoke-opencode.lock.yml @@ -1163,12 +1163,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1195,6 +1196,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1225,7 +1230,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1293,12 +1298,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1361,6 +1369,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-otel-backends.lock.yml b/.github/workflows/smoke-otel-backends.lock.yml index fd19c4e655c..114c8b27ede 100644 --- a/.github/workflows/smoke-otel-backends.lock.yml +++ b/.github/workflows/smoke-otel-backends.lock.yml @@ -1302,12 +1302,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1334,6 +1335,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1364,7 +1369,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1432,12 +1437,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1500,6 +1508,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-pi.lock.yml b/.github/workflows/smoke-pi.lock.yml index e0ef434a2d7..e58b535ec34 100644 --- a/.github/workflows/smoke-pi.lock.yml +++ b/.github/workflows/smoke-pi.lock.yml @@ -1216,12 +1216,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1248,6 +1249,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1278,7 +1283,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1346,12 +1351,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1414,6 +1422,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-project.lock.yml b/.github/workflows/smoke-project.lock.yml index 09fc94dae47..dd55302223a 100644 --- a/.github/workflows/smoke-project.lock.yml +++ b/.github/workflows/smoke-project.lock.yml @@ -1386,12 +1386,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1418,6 +1419,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1448,7 +1453,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1516,12 +1521,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1584,6 +1592,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-service-ports.lock.yml b/.github/workflows/smoke-service-ports.lock.yml index 0dee877f7ce..18278b4151c 100644 --- a/.github/workflows/smoke-service-ports.lock.yml +++ b/.github/workflows/smoke-service-ports.lock.yml @@ -1130,12 +1130,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1162,6 +1163,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1192,7 +1197,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1260,12 +1265,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1328,6 +1336,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-temporary-id.lock.yml b/.github/workflows/smoke-temporary-id.lock.yml index 75c9d569d67..ea71d210227 100644 --- a/.github/workflows/smoke-temporary-id.lock.yml +++ b/.github/workflows/smoke-temporary-id.lock.yml @@ -1231,12 +1231,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1263,6 +1264,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1293,7 +1298,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1361,12 +1366,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1429,6 +1437,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-test-tools.lock.yml b/.github/workflows/smoke-test-tools.lock.yml index 07e7e765cd3..e5a6ea3cfee 100644 --- a/.github/workflows/smoke-test-tools.lock.yml +++ b/.github/workflows/smoke-test-tools.lock.yml @@ -1162,12 +1162,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1194,6 +1195,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1224,7 +1229,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1292,12 +1297,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1360,6 +1368,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-update-cross-repo-pr.lock.yml b/.github/workflows/smoke-update-cross-repo-pr.lock.yml index 3f1817a34ec..2c15c3b8efa 100644 --- a/.github/workflows/smoke-update-cross-repo-pr.lock.yml +++ b/.github/workflows/smoke-update-cross-repo-pr.lock.yml @@ -1290,12 +1290,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1322,6 +1323,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1352,7 +1357,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1420,12 +1425,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1488,6 +1496,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-workflow-call-with-inputs.lock.yml b/.github/workflows/smoke-workflow-call-with-inputs.lock.yml index a2b904e2877..9def7dd8b7c 100644 --- a/.github/workflows/smoke-workflow-call-with-inputs.lock.yml +++ b/.github/workflows/smoke-workflow-call-with-inputs.lock.yml @@ -1187,12 +1187,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1219,6 +1220,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1249,7 +1254,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1317,12 +1322,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1385,6 +1393,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/smoke-workflow-call.lock.yml b/.github/workflows/smoke-workflow-call.lock.yml index 39e3ac236c9..a036bb552ea 100644 --- a/.github/workflows/smoke-workflow-call.lock.yml +++ b/.github/workflows/smoke-workflow-call.lock.yml @@ -1176,12 +1176,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1208,6 +1209,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1238,7 +1243,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1306,12 +1311,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1374,6 +1382,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/spec-enforcer.lock.yml b/.github/workflows/spec-enforcer.lock.yml index b5ac61e92d3..7ee758698cc 100644 --- a/.github/workflows/spec-enforcer.lock.yml +++ b/.github/workflows/spec-enforcer.lock.yml @@ -1122,12 +1122,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1154,6 +1155,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1184,7 +1189,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1252,12 +1257,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1320,6 +1328,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/spec-extractor.lock.yml b/.github/workflows/spec-extractor.lock.yml index 3f1b882e332..40cda971a58 100644 --- a/.github/workflows/spec-extractor.lock.yml +++ b/.github/workflows/spec-extractor.lock.yml @@ -1215,12 +1215,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1247,6 +1248,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1277,7 +1282,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1345,12 +1350,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1413,6 +1421,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/spec-librarian.lock.yml b/.github/workflows/spec-librarian.lock.yml index 79bd78b0688..5da845e8235 100644 --- a/.github/workflows/spec-librarian.lock.yml +++ b/.github/workflows/spec-librarian.lock.yml @@ -1176,12 +1176,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1208,6 +1209,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1238,7 +1243,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1306,12 +1311,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1374,6 +1382,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/stale-pr-cleanup.lock.yml b/.github/workflows/stale-pr-cleanup.lock.yml index 0a03f34b1d9..fd251f92776 100644 --- a/.github/workflows/stale-pr-cleanup.lock.yml +++ b/.github/workflows/stale-pr-cleanup.lock.yml @@ -1123,12 +1123,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1155,6 +1156,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1185,7 +1190,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1253,12 +1258,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1321,6 +1329,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/stale-repo-identifier.lock.yml b/.github/workflows/stale-repo-identifier.lock.yml index 97c062775ac..02a7bfee82f 100644 --- a/.github/workflows/stale-repo-identifier.lock.yml +++ b/.github/workflows/stale-repo-identifier.lock.yml @@ -1319,12 +1319,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1351,6 +1352,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1381,7 +1386,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1449,12 +1454,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1517,6 +1525,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/static-analysis-report.lock.yml b/.github/workflows/static-analysis-report.lock.yml index d6d792d8037..9c55c36b3c2 100644 --- a/.github/workflows/static-analysis-report.lock.yml +++ b/.github/workflows/static-analysis-report.lock.yml @@ -1332,12 +1332,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1364,6 +1365,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1394,7 +1399,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1462,12 +1467,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1530,6 +1538,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/step-name-alignment.lock.yml b/.github/workflows/step-name-alignment.lock.yml index afb6709f0f0..b2eefd398de 100644 --- a/.github/workflows/step-name-alignment.lock.yml +++ b/.github/workflows/step-name-alignment.lock.yml @@ -1218,12 +1218,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1250,6 +1251,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1280,7 +1285,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1348,12 +1353,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1416,6 +1424,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/sub-issue-closer.lock.yml b/.github/workflows/sub-issue-closer.lock.yml index 065e2f6177e..1ee8e2e4999 100644 --- a/.github/workflows/sub-issue-closer.lock.yml +++ b/.github/workflows/sub-issue-closer.lock.yml @@ -1124,12 +1124,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1156,6 +1157,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1186,7 +1191,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1254,12 +1259,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1322,6 +1330,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/super-linter.lock.yml b/.github/workflows/super-linter.lock.yml index 955fa37cb60..5bb97fc4ecc 100644 --- a/.github/workflows/super-linter.lock.yml +++ b/.github/workflows/super-linter.lock.yml @@ -1152,12 +1152,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1184,6 +1185,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1214,7 +1219,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1282,12 +1287,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1350,6 +1358,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/technical-doc-writer.lock.yml b/.github/workflows/technical-doc-writer.lock.yml index 1c419a5ef74..c8f91974550 100644 --- a/.github/workflows/technical-doc-writer.lock.yml +++ b/.github/workflows/technical-doc-writer.lock.yml @@ -1242,12 +1242,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1274,6 +1275,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1304,7 +1309,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1372,12 +1377,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1440,6 +1448,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/terminal-stylist.lock.yml b/.github/workflows/terminal-stylist.lock.yml index 50acac163cf..b03f34a8dfa 100644 --- a/.github/workflows/terminal-stylist.lock.yml +++ b/.github/workflows/terminal-stylist.lock.yml @@ -1114,12 +1114,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1146,6 +1147,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1176,7 +1181,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1244,12 +1249,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1312,6 +1320,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/test-create-pr-error-handling.lock.yml b/.github/workflows/test-create-pr-error-handling.lock.yml index 385249c86b2..e735b3598d2 100644 --- a/.github/workflows/test-create-pr-error-handling.lock.yml +++ b/.github/workflows/test-create-pr-error-handling.lock.yml @@ -1199,12 +1199,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1231,6 +1232,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1261,7 +1266,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1329,12 +1334,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1397,6 +1405,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/test-dispatcher.lock.yml b/.github/workflows/test-dispatcher.lock.yml index bb885ca3248..9e7231c9339 100644 --- a/.github/workflows/test-dispatcher.lock.yml +++ b/.github/workflows/test-dispatcher.lock.yml @@ -1079,12 +1079,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1111,6 +1112,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1141,7 +1146,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1209,12 +1214,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1277,6 +1285,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/test-project-url-default.lock.yml b/.github/workflows/test-project-url-default.lock.yml index d6e0ab293d1..eab10b1e6f9 100644 --- a/.github/workflows/test-project-url-default.lock.yml +++ b/.github/workflows/test-project-url-default.lock.yml @@ -1125,12 +1125,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1157,6 +1158,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1187,7 +1192,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1255,12 +1260,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1323,6 +1331,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/test-quality-sentinel.lock.yml b/.github/workflows/test-quality-sentinel.lock.yml index 8f72df0cad8..42d46637de7 100644 --- a/.github/workflows/test-quality-sentinel.lock.yml +++ b/.github/workflows/test-quality-sentinel.lock.yml @@ -1192,12 +1192,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1224,6 +1225,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1254,7 +1259,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1322,12 +1327,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1390,6 +1398,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/test-workflow.lock.yml b/.github/workflows/test-workflow.lock.yml index 914946bb219..cfa94c724d5 100644 --- a/.github/workflows/test-workflow.lock.yml +++ b/.github/workflows/test-workflow.lock.yml @@ -1002,12 +1002,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1034,6 +1035,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1064,7 +1069,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1132,12 +1137,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1200,6 +1208,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/tidy.lock.yml b/.github/workflows/tidy.lock.yml index 311fccb3774..29267e1b35b 100644 --- a/.github/workflows/tidy.lock.yml +++ b/.github/workflows/tidy.lock.yml @@ -1221,12 +1221,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1253,6 +1254,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1283,7 +1288,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1351,12 +1356,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1419,6 +1427,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/typist.lock.yml b/.github/workflows/typist.lock.yml index 268116d72b5..e771ce385a0 100644 --- a/.github/workflows/typist.lock.yml +++ b/.github/workflows/typist.lock.yml @@ -1239,12 +1239,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1271,6 +1272,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1301,7 +1306,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1369,12 +1374,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1437,6 +1445,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/ubuntu-image-analyzer.lock.yml b/.github/workflows/ubuntu-image-analyzer.lock.yml index 704a6d36e78..14906652379 100644 --- a/.github/workflows/ubuntu-image-analyzer.lock.yml +++ b/.github/workflows/ubuntu-image-analyzer.lock.yml @@ -1135,12 +1135,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1167,6 +1168,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1197,7 +1202,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1265,12 +1270,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1333,6 +1341,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/uk-ai-operational-resilience.lock.yml b/.github/workflows/uk-ai-operational-resilience.lock.yml index 2033fb26443..f295dbae6f6 100644 --- a/.github/workflows/uk-ai-operational-resilience.lock.yml +++ b/.github/workflows/uk-ai-operational-resilience.lock.yml @@ -1115,12 +1115,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1147,6 +1148,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1177,7 +1182,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1245,12 +1250,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1313,6 +1321,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/unbloat-docs.lock.yml b/.github/workflows/unbloat-docs.lock.yml index 2195eb2504a..a9162b13ffd 100644 --- a/.github/workflows/unbloat-docs.lock.yml +++ b/.github/workflows/unbloat-docs.lock.yml @@ -1213,12 +1213,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1245,6 +1246,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1275,7 +1280,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1343,12 +1348,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1411,6 +1419,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/update-astro.lock.yml b/.github/workflows/update-astro.lock.yml index d1e1b5b9228..8fecbe5d9e2 100644 --- a/.github/workflows/update-astro.lock.yml +++ b/.github/workflows/update-astro.lock.yml @@ -1160,12 +1160,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1192,6 +1193,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1222,7 +1227,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1290,12 +1295,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1358,6 +1366,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/video-analyzer.lock.yml b/.github/workflows/video-analyzer.lock.yml index 3a55047212d..ad582a8e4bd 100644 --- a/.github/workflows/video-analyzer.lock.yml +++ b/.github/workflows/video-analyzer.lock.yml @@ -1102,12 +1102,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1134,6 +1135,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1164,7 +1169,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1232,12 +1237,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1300,6 +1308,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/visual-regression-checker.lock.yml b/.github/workflows/visual-regression-checker.lock.yml index 1a019ce7d5a..5776da729e2 100644 --- a/.github/workflows/visual-regression-checker.lock.yml +++ b/.github/workflows/visual-regression-checker.lock.yml @@ -1172,12 +1172,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1204,6 +1205,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1234,7 +1239,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1302,12 +1307,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1370,6 +1378,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/weekly-blog-post-writer.lock.yml b/.github/workflows/weekly-blog-post-writer.lock.yml index 8d6abc38351..acc256ac414 100644 --- a/.github/workflows/weekly-blog-post-writer.lock.yml +++ b/.github/workflows/weekly-blog-post-writer.lock.yml @@ -1300,12 +1300,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1332,6 +1333,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1362,7 +1367,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1430,12 +1435,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1498,6 +1506,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/weekly-editors-health-check.lock.yml b/.github/workflows/weekly-editors-health-check.lock.yml index 4866728801f..003d8405e8e 100644 --- a/.github/workflows/weekly-editors-health-check.lock.yml +++ b/.github/workflows/weekly-editors-health-check.lock.yml @@ -1169,12 +1169,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1201,6 +1202,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1231,7 +1236,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1299,12 +1304,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1367,6 +1375,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/weekly-issue-summary.lock.yml b/.github/workflows/weekly-issue-summary.lock.yml index ab5b3b7ed24..36f9217f50f 100644 --- a/.github/workflows/weekly-issue-summary.lock.yml +++ b/.github/workflows/weekly-issue-summary.lock.yml @@ -1157,12 +1157,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1189,6 +1190,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1219,7 +1224,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1287,12 +1292,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1355,6 +1363,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/weekly-safe-outputs-spec-review.lock.yml b/.github/workflows/weekly-safe-outputs-spec-review.lock.yml index ba1de1cd350..97fa3f8b22b 100644 --- a/.github/workflows/weekly-safe-outputs-spec-review.lock.yml +++ b/.github/workflows/weekly-safe-outputs-spec-review.lock.yml @@ -1092,12 +1092,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1124,6 +1125,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1154,7 +1159,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1222,12 +1227,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1290,6 +1298,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/workflow-generator.lock.yml b/.github/workflows/workflow-generator.lock.yml index 544ff86c966..7a72dcbedc8 100644 --- a/.github/workflows/workflow-generator.lock.yml +++ b/.github/workflows/workflow-generator.lock.yml @@ -1165,12 +1165,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1197,6 +1198,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1227,7 +1232,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1295,12 +1300,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1363,6 +1371,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/workflow-health-manager.lock.yml b/.github/workflows/workflow-health-manager.lock.yml index c6d84164713..fd479f7e791 100644 --- a/.github/workflows/workflow-health-manager.lock.yml +++ b/.github/workflows/workflow-health-manager.lock.yml @@ -1211,12 +1211,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1243,6 +1244,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1273,7 +1278,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1341,12 +1346,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1409,6 +1417,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/workflow-normalizer.lock.yml b/.github/workflows/workflow-normalizer.lock.yml index 9c862caefd1..545897b8f5c 100644 --- a/.github/workflows/workflow-normalizer.lock.yml +++ b/.github/workflows/workflow-normalizer.lock.yml @@ -1175,12 +1175,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1207,6 +1208,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1237,7 +1242,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1305,12 +1310,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1373,6 +1381,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json diff --git a/.github/workflows/workflow-skill-extractor.lock.yml b/.github/workflows/workflow-skill-extractor.lock.yml index a0c97942eca..2c9235c539e 100644 --- a/.github/workflows/workflow-skill-extractor.lock.yml +++ b/.github/workflows/workflow-skill-extractor.lock.yml @@ -1146,12 +1146,13 @@ jobs: run: | mkdir -p /tmp/gh-aw/usage/agent /tmp/gh-aw/usage/detection echo "Usage artifact source file status:" - for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do + for file in /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/threat-detection/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl; do [ -f "$file" ] && echo "FOUND: $file" || echo "MISSING: $file" done [ -f /tmp/gh-aw/aw-info.jsonl ] && cp /tmp/gh-aw/aw-info.jsonl /tmp/gh-aw/usage/aw-info.jsonl || true [ -f /tmp/gh-aw/agent_usage.jsonl ] && cp /tmp/gh-aw/agent_usage.jsonl /tmp/gh-aw/usage/agent_usage.jsonl || true [ -f /tmp/gh-aw/detection_usage.jsonl ] && cp /tmp/gh-aw/detection_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl || true + [ -f /tmp/gh-aw/github_rate_limits.jsonl ] && cp /tmp/gh-aw/github_rate_limits.jsonl /tmp/gh-aw/usage/github_rate_limits.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall-audit-logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/logs/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true [ -f /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl ] && cp /tmp/gh-aw/sandbox/firewall/audit/api-proxy-logs/token-usage.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl || true @@ -1178,6 +1179,10 @@ jobs: SQUID_DECISION_INDEX = 7 firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} + def is_allowed_decision(decision: str) -> bool: + base = decision.split('/', 1)[0].strip().upper() + return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') + firewall_paths = [ '/tmp/gh-aw/sandbox/firewall/logs/*.log', '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', @@ -1208,7 +1213,7 @@ jobs: allowed = code in (200, 206, 304) except ValueError: allowed = False - if not allowed and any(marker in decision for marker in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')): + if not allowed and is_allowed_decision(decision): allowed = True if allowed: firewall['allowed_requests'] += 1 @@ -1276,12 +1281,15 @@ jobs: summary['session'] = session gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl', - ] + gateway_paths = [] + for modern_path, legacy_path in [ + ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), + ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), + ]: + if os.path.exists(modern_path): + gateway_paths.append(modern_path) + elif os.path.exists(legacy_path): + gateway_paths.append(legacy_path) for gateway_path in gateway_paths: if not os.path.exists(gateway_path): continue @@ -1344,6 +1352,7 @@ jobs: /tmp/gh-aw/usage/aw-info.jsonl /tmp/gh-aw/usage/agent_usage.jsonl /tmp/gh-aw/usage/detection_usage.jsonl + /tmp/gh-aw/usage/github_rate_limits.jsonl /tmp/gh-aw/usage/agent/token_usage.jsonl /tmp/gh-aw/usage/detection/token_usage.jsonl /tmp/gh-aw/usage/activity/summary.json From 01ecbfbb9aceea4efeed0a377e0ebd8baae902f1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 21 Jun 2026 02:45:10 +0000 Subject: [PATCH 14/17] Replace Python aggregation script with JavaScript Moved usage activity summary generation from inline Python heredoc to a standalone JavaScript file (actions/setup/js/generate_usage_activity_summary.cjs) per user request. Updated notify_comment.go to call the new script via node, and adjusted tests to check for the JavaScript file instead of Python markers. All workflows recompiled successfully. Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/ab-testing-advisor.lock.yml | 180 +---------- .github/workflows/ace-editor.lock.yml | 180 +---------- .../agent-performance-analyzer.lock.yml | 180 +---------- .../workflows/agent-persona-explorer.lock.yml | 180 +---------- .../workflows/agentic-token-audit.lock.yml | 180 +---------- .../agentic-token-optimizer.lock.yml | 180 +---------- .../agentic-token-trend-audit.lock.yml | 180 +---------- .github/workflows/ai-moderator.lock.yml | 180 +---------- .../workflows/api-consumption-report.lock.yml | 180 +---------- .github/workflows/approach-validator.lock.yml | 180 +---------- .github/workflows/archie.lock.yml | 180 +---------- .../workflows/architecture-guardian.lock.yml | 180 +---------- .github/workflows/artifacts-summary.lock.yml | 180 +---------- .github/workflows/audit-workflows.lock.yml | 180 +---------- .github/workflows/auto-triage-issues.lock.yml | 180 +---------- .github/workflows/avenger.lock.yml | 180 +---------- .../aw-failure-investigator.lock.yml | 180 +---------- .github/workflows/blog-auditor.lock.yml | 180 +---------- .github/workflows/bot-detection.lock.yml | 180 +---------- .github/workflows/brave.lock.yml | 180 +---------- .../breaking-change-checker.lock.yml | 180 +---------- .github/workflows/changeset.lock.yml | 180 +---------- .../workflows/chaos-pr-bundle-fuzzer.lock.yml | 180 +---------- .github/workflows/ci-coach.lock.yml | 180 +---------- .github/workflows/ci-doctor.lock.yml | 180 +---------- .../claude-code-user-docs-review.lock.yml | 180 +---------- .../cli-consistency-checker.lock.yml | 180 +---------- .../workflows/cli-version-checker.lock.yml | 180 +---------- .github/workflows/cloclo.lock.yml | 180 +---------- .../workflows/code-scanning-fixer.lock.yml | 180 +---------- .github/workflows/code-simplifier.lock.yml | 180 +---------- .../codex-github-remote-mcp-test.lock.yml | 180 +---------- .../commit-changes-analyzer.lock.yml | 180 +---------- .../constraint-solving-potd.lock.yml | 180 +---------- .github/workflows/contribution-check.lock.yml | 180 +---------- .../workflows/copilot-agent-analysis.lock.yml | 180 +---------- .../copilot-centralization-drilldown.lock.yml | 180 +---------- .../copilot-centralization-optimizer.lock.yml | 180 +---------- .../copilot-cli-deep-research.lock.yml | 180 +---------- .github/workflows/copilot-opt.lock.yml | 180 +---------- .../copilot-pr-merged-report.lock.yml | 180 +---------- .../copilot-pr-nlp-analysis.lock.yml | 180 +---------- .../copilot-pr-prompt-analysis.lock.yml | 180 +---------- .../copilot-session-insights.lock.yml | 180 +---------- .github/workflows/craft.lock.yml | 180 +---------- ...aily-agent-of-the-day-blog-writer.lock.yml | 180 +---------- .../daily-agentrx-trace-optimizer.lock.yml | 180 +---------- .../daily-ambient-context-optimizer.lock.yml | 180 +---------- .../daily-architecture-diagram.lock.yml | 180 +---------- .../daily-assign-issue-to-user.lock.yml | 180 +---------- ...strostylelite-markdown-spellcheck.lock.yml | 180 +---------- ...daily-aw-cross-repo-compile-check.lock.yml | 180 +---------- ...daily-awf-spec-compiler-surfacing.lock.yml | 180 +---------- .../workflows/daily-byok-ollama-test.lock.yml | 180 +---------- .../daily-cache-strategy-analyzer.lock.yml | 180 +---------- .../daily-caveman-optimizer.lock.yml | 180 +---------- .github/workflows/daily-choice-test.lock.yml | 180 +---------- .../workflows/daily-cli-performance.lock.yml | 180 +---------- .../workflows/daily-cli-tools-tester.lock.yml | 180 +---------- .github/workflows/daily-code-metrics.lock.yml | 180 +---------- .../daily-community-attribution.lock.yml | 180 +---------- .../workflows/daily-compiler-quality.lock.yml | 180 +---------- ...ly-compiler-threat-spec-optimizer.lock.yml | 180 +---------- .../daily-credit-limit-test.lock.yml | 180 +---------- .github/workflows/daily-doc-healer.lock.yml | 180 +---------- .github/workflows/daily-doc-updater.lock.yml | 180 +---------- .../daily-experiment-report.lock.yml | 180 +---------- .github/workflows/daily-fact.lock.yml | 180 +---------- .github/workflows/daily-file-diet.lock.yml | 180 +---------- .../workflows/daily-firewall-report.lock.yml | 180 +---------- .../daily-formal-spec-verifier.lock.yml | 180 +---------- .../workflows/daily-function-namer.lock.yml | 180 +---------- .../workflows/daily-geo-optimizer.lock.yml | 180 +---------- .github/workflows/daily-hippo-learn.lock.yml | 180 +---------- .../workflows/daily-issues-report.lock.yml | 180 +---------- .../daily-malicious-code-scan.lock.yml | 180 +---------- .../daily-max-ai-credits-test.lock.yml | 180 +---------- .../daily-mcp-concurrency-analysis.lock.yml | 180 +---------- .../workflows/daily-model-inventory.lock.yml | 180 +---------- .../daily-multi-device-docs-tester.lock.yml | 180 +---------- .github/workflows/daily-news.lock.yml | 180 +---------- .../daily-observability-report.lock.yml | 180 +---------- .../daily-performance-summary.lock.yml | 180 +---------- .github/workflows/daily-regulatory.lock.yml | 180 +---------- .../daily-reliability-review.lock.yml | 180 +---------- .../daily-rendering-scripts-verifier.lock.yml | 180 +---------- .../workflows/daily-repo-chronicle.lock.yml | 180 +---------- .../daily-safe-output-integrator.lock.yml | 180 +---------- .../daily-safe-output-optimizer.lock.yml | 180 +---------- .../daily-safe-outputs-conformance.lock.yml | 180 +---------- .../daily-safeoutputs-git-simulator.lock.yml | 180 +---------- .../workflows/daily-secrets-analysis.lock.yml | 180 +---------- .../daily-security-observability.lock.yml | 180 +---------- .../daily-security-red-team.lock.yml | 180 +---------- .github/workflows/daily-semgrep-scan.lock.yml | 180 +---------- .../workflows/daily-sentrux-report.lock.yml | 180 +---------- .../workflows/daily-skill-optimizer.lock.yml | 180 +---------- .../daily-spdd-spec-planner.lock.yml | 180 +---------- .../daily-syntax-error-quality.lock.yml | 180 +---------- .../daily-team-evolution-insights.lock.yml | 180 +---------- .github/workflows/daily-team-status.lock.yml | 180 +---------- .../daily-testify-uber-super-expert.lock.yml | 180 +---------- .../daily-token-consumption-report.lock.yml | 180 +---------- ...dows-terminal-integration-builder.lock.yml | 180 +---------- .../workflows/daily-workflow-updater.lock.yml | 180 +---------- .../dataflow-pr-discussion-dataset.lock.yml | 180 +---------- .github/workflows/dead-code-remover.lock.yml | 180 +---------- .github/workflows/deep-report.lock.yml | 180 +---------- .github/workflows/delight.lock.yml | 180 +---------- .github/workflows/dependabot-burner.lock.yml | 180 +---------- .../workflows/dependabot-go-checker.lock.yml | 180 +---------- .github/workflows/dependabot-repair.lock.yml | 180 +---------- .../deployment-incident-monitor.lock.yml | 180 +---------- .../workflows/design-decision-gate.lock.yml | 180 +---------- .../workflows/designer-drift-audit.lock.yml | 180 +---------- .github/workflows/dev-hawk.lock.yml | 180 +---------- .github/workflows/dev.lock.yml | 180 +---------- .../developer-docs-consolidator.lock.yml | 180 +---------- .github/workflows/dictation-prompt.lock.yml | 180 +---------- .../workflows/discussion-task-miner.lock.yml | 180 +---------- .github/workflows/docs-noob-tester.lock.yml | 180 +---------- .github/workflows/draft-pr-cleanup.lock.yml | 180 +---------- .../duplicate-code-detector.lock.yml | 180 +---------- .../example-failure-category-filter.lock.yml | 180 +---------- .../example-permissions-warning.lock.yml | 180 +---------- .../example-workflow-analyzer.lock.yml | 180 +---------- .github/workflows/firewall-escape.lock.yml | 180 +---------- .github/workflows/firewall.lock.yml | 180 +---------- .../workflows/functional-pragmatist.lock.yml | 180 +---------- .../github-mcp-structural-analysis.lock.yml | 180 +---------- .../github-mcp-tools-report.lock.yml | 180 +---------- .../github-remote-mcp-auth-test.lock.yml | 180 +---------- .../workflows/glossary-maintainer.lock.yml | 180 +---------- .github/workflows/go-fan.lock.yml | 180 +---------- .github/workflows/go-logger.lock.yml | 180 +---------- .../workflows/go-pattern-detector.lock.yml | 180 +---------- .github/workflows/gpclean.lock.yml | 180 +---------- .github/workflows/grumpy-reviewer.lock.yml | 180 +---------- .github/workflows/hippo-embed.lock.yml | 180 +---------- .github/workflows/hourly-ci-cleaner.lock.yml | 180 +---------- .../workflows/instructions-janitor.lock.yml | 180 +---------- .github/workflows/issue-arborist.lock.yml | 180 +---------- .github/workflows/issue-monster.lock.yml | 180 +---------- .github/workflows/issue-triage-agent.lock.yml | 180 +---------- .github/workflows/jsweep.lock.yml | 180 +---------- .../workflows/layout-spec-maintainer.lock.yml | 180 +---------- .github/workflows/lint-monster.lock.yml | 180 +---------- .github/workflows/linter-miner.lock.yml | 180 +---------- .github/workflows/lockfile-stats.lock.yml | 180 +---------- .../mattpocock-skills-reviewer.lock.yml | 180 +---------- .github/workflows/mcp-inspector.lock.yml | 180 +---------- .github/workflows/mergefest.lock.yml | 180 +---------- .github/workflows/metrics-collector.lock.yml | 180 +---------- .github/workflows/necromancer.lock.yml | 180 +---------- .../workflows/notion-issue-summary.lock.yml | 180 +---------- .../objective-impact-report.lock.yml | 180 +---------- .github/workflows/org-health-report.lock.yml | 180 +---------- .github/workflows/outcome-collector.lock.yml | 180 +---------- .github/workflows/pdf-summary.lock.yml | 180 +---------- .github/workflows/plan.lock.yml | 180 +---------- .github/workflows/poem-bot.lock.yml | 180 +---------- .github/workflows/portfolio-analyst.lock.yml | 180 +---------- .../pr-code-quality-reviewer.lock.yml | 180 +---------- .../workflows/pr-description-caveman.lock.yml | 180 +---------- .../workflows/pr-nitpick-reviewer.lock.yml | 180 +---------- .github/workflows/pr-sous-chef.lock.yml | 180 +---------- .github/workflows/pr-triage-agent.lock.yml | 180 +---------- .../prompt-clustering-analysis.lock.yml | 180 +---------- .github/workflows/python-data-charts.lock.yml | 180 +---------- .github/workflows/q.lock.yml | 180 +---------- .../workflows/refactoring-cadence.lock.yml | 180 +---------- .github/workflows/refiner.lock.yml | 180 +---------- .github/workflows/release.lock.yml | 180 +---------- .../workflows/repo-audit-analyzer.lock.yml | 180 +---------- .github/workflows/repo-tree-map.lock.yml | 180 +---------- .../repository-quality-improver.lock.yml | 180 +---------- .github/workflows/research.lock.yml | 180 +---------- .github/workflows/ruflo-backed-task.lock.yml | 180 +---------- .github/workflows/safe-output-health.lock.yml | 180 +---------- .../schema-consistency-checker.lock.yml | 180 +---------- .../schema-feature-coverage.lock.yml | 180 +---------- .github/workflows/scout.lock.yml | 180 +---------- .../workflows/security-compliance.lock.yml | 180 +---------- .github/workflows/security-review.lock.yml | 180 +---------- .../semantic-function-refactor.lock.yml | 180 +---------- .github/workflows/sergo.lock.yml | 180 +---------- .github/workflows/skillet.lock.yml | 180 +---------- .../workflows/slide-deck-maintainer.lock.yml | 180 +---------- .../workflows/smoke-agent-all-merged.lock.yml | 180 +---------- .../workflows/smoke-agent-all-none.lock.yml | 180 +---------- .../smoke-agent-public-approved.lock.yml | 180 +---------- .../smoke-agent-public-none.lock.yml | 180 +---------- .../smoke-agent-scoped-approved.lock.yml | 180 +---------- .github/workflows/smoke-antigravity.lock.yml | 180 +---------- .../workflows/smoke-call-workflow.lock.yml | 180 +---------- .github/workflows/smoke-ci.lock.yml | 180 +---------- .github/workflows/smoke-claude.lock.yml | 180 +---------- .github/workflows/smoke-codex.lock.yml | 180 +---------- .../smoke-copilot-aoai-apikey.lock.yml | 180 +---------- .../smoke-copilot-aoai-entra.lock.yml | 180 +---------- .github/workflows/smoke-copilot-arm.lock.yml | 180 +---------- .github/workflows/smoke-copilot-sdk.lock.yml | 180 +---------- .github/workflows/smoke-copilot.lock.yml | 180 +---------- .../smoke-create-cross-repo-pr.lock.yml | 180 +---------- .github/workflows/smoke-crush.lock.yml | 180 +---------- .github/workflows/smoke-gemini.lock.yml | 180 +---------- .github/workflows/smoke-multi-pr.lock.yml | 180 +---------- .github/workflows/smoke-opencode.lock.yml | 180 +---------- .../workflows/smoke-otel-backends.lock.yml | 180 +---------- .github/workflows/smoke-pi.lock.yml | 180 +---------- .github/workflows/smoke-project.lock.yml | 180 +---------- .../workflows/smoke-service-ports.lock.yml | 180 +---------- .github/workflows/smoke-temporary-id.lock.yml | 180 +---------- .github/workflows/smoke-test-tools.lock.yml | 180 +---------- .../smoke-update-cross-repo-pr.lock.yml | 180 +---------- .../smoke-workflow-call-with-inputs.lock.yml | 180 +---------- .../workflows/smoke-workflow-call.lock.yml | 180 +---------- .github/workflows/spec-enforcer.lock.yml | 180 +---------- .github/workflows/spec-extractor.lock.yml | 180 +---------- .github/workflows/spec-librarian.lock.yml | 180 +---------- .github/workflows/stale-pr-cleanup.lock.yml | 180 +---------- .../workflows/stale-repo-identifier.lock.yml | 180 +---------- .../workflows/static-analysis-report.lock.yml | 180 +---------- .../workflows/step-name-alignment.lock.yml | 180 +---------- .github/workflows/sub-issue-closer.lock.yml | 180 +---------- .github/workflows/super-linter.lock.yml | 180 +---------- .../workflows/technical-doc-writer.lock.yml | 180 +---------- .github/workflows/terminal-stylist.lock.yml | 180 +---------- .../test-create-pr-error-handling.lock.yml | 180 +---------- .github/workflows/test-dispatcher.lock.yml | 180 +---------- .../test-project-url-default.lock.yml | 180 +---------- .../workflows/test-quality-sentinel.lock.yml | 180 +---------- .github/workflows/test-workflow.lock.yml | 180 +---------- .github/workflows/tidy.lock.yml | 180 +---------- .github/workflows/typist.lock.yml | 180 +---------- .../workflows/ubuntu-image-analyzer.lock.yml | 180 +---------- .../uk-ai-operational-resilience.lock.yml | 180 +---------- .github/workflows/unbloat-docs.lock.yml | 180 +---------- .github/workflows/update-astro.lock.yml | 180 +---------- .github/workflows/video-analyzer.lock.yml | 180 +---------- .../visual-regression-checker.lock.yml | 180 +---------- .../weekly-blog-post-writer.lock.yml | 180 +---------- .../weekly-editors-health-check.lock.yml | 180 +---------- .../workflows/weekly-issue-summary.lock.yml | 180 +---------- .../weekly-safe-outputs-spec-review.lock.yml | 180 +---------- .github/workflows/workflow-generator.lock.yml | 180 +---------- .../workflow-health-manager.lock.yml | 180 +---------- .../workflows/workflow-normalizer.lock.yml | 180 +---------- .../workflow-skill-extractor.lock.yml | 180 +---------- .../js/generate_usage_activity_summary.cjs | 298 ++++++++++++++++++ pkg/workflow/notify_comment.go | 180 +---------- pkg/workflow/notify_comment_test.go | 8 +- 252 files changed, 549 insertions(+), 44757 deletions(-) create mode 100644 actions/setup/js/generate_usage_activity_summary.cjs diff --git a/.github/workflows/ab-testing-advisor.lock.yml b/.github/workflows/ab-testing-advisor.lock.yml index ac6198ab0ea..7781a5bb892 100644 --- a/.github/workflows/ab-testing-advisor.lock.yml +++ b/.github/workflows/ab-testing-advisor.lock.yml @@ -1114,185 +1114,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/ace-editor.lock.yml b/.github/workflows/ace-editor.lock.yml index 9b0cf11bd66..319a24b8167 100644 --- a/.github/workflows/ace-editor.lock.yml +++ b/.github/workflows/ace-editor.lock.yml @@ -1052,185 +1052,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/agent-performance-analyzer.lock.yml b/.github/workflows/agent-performance-analyzer.lock.yml index 52f5df47152..f80c653cc90 100644 --- a/.github/workflows/agent-performance-analyzer.lock.yml +++ b/.github/workflows/agent-performance-analyzer.lock.yml @@ -1321,185 +1321,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/agent-persona-explorer.lock.yml b/.github/workflows/agent-persona-explorer.lock.yml index c33c384ab5a..4d009d35a16 100644 --- a/.github/workflows/agent-persona-explorer.lock.yml +++ b/.github/workflows/agent-persona-explorer.lock.yml @@ -1238,185 +1238,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/agentic-token-audit.lock.yml b/.github/workflows/agentic-token-audit.lock.yml index 9454917207d..65b3e8a78a9 100644 --- a/.github/workflows/agentic-token-audit.lock.yml +++ b/.github/workflows/agentic-token-audit.lock.yml @@ -1252,185 +1252,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/agentic-token-optimizer.lock.yml b/.github/workflows/agentic-token-optimizer.lock.yml index c6cd5563e04..72f2c6cc92f 100644 --- a/.github/workflows/agentic-token-optimizer.lock.yml +++ b/.github/workflows/agentic-token-optimizer.lock.yml @@ -1119,185 +1119,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/agentic-token-trend-audit.lock.yml b/.github/workflows/agentic-token-trend-audit.lock.yml index 801121518b6..615e4fa5ffb 100644 --- a/.github/workflows/agentic-token-trend-audit.lock.yml +++ b/.github/workflows/agentic-token-trend-audit.lock.yml @@ -1209,185 +1209,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/ai-moderator.lock.yml b/.github/workflows/ai-moderator.lock.yml index ad5e35b39e9..3afe88c419a 100644 --- a/.github/workflows/ai-moderator.lock.yml +++ b/.github/workflows/ai-moderator.lock.yml @@ -1242,185 +1242,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/api-consumption-report.lock.yml b/.github/workflows/api-consumption-report.lock.yml index e54ff0dd993..239d3367379 100644 --- a/.github/workflows/api-consumption-report.lock.yml +++ b/.github/workflows/api-consumption-report.lock.yml @@ -1591,185 +1591,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/approach-validator.lock.yml b/.github/workflows/approach-validator.lock.yml index e89c160a811..81f7b1fdc5b 100644 --- a/.github/workflows/approach-validator.lock.yml +++ b/.github/workflows/approach-validator.lock.yml @@ -1295,185 +1295,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/archie.lock.yml b/.github/workflows/archie.lock.yml index 6eb5c1b0b19..0ad7d80133b 100644 --- a/.github/workflows/archie.lock.yml +++ b/.github/workflows/archie.lock.yml @@ -1180,185 +1180,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/architecture-guardian.lock.yml b/.github/workflows/architecture-guardian.lock.yml index cbf1c6a64f7..5f1b0bfbc6b 100644 --- a/.github/workflows/architecture-guardian.lock.yml +++ b/.github/workflows/architecture-guardian.lock.yml @@ -1188,185 +1188,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/artifacts-summary.lock.yml b/.github/workflows/artifacts-summary.lock.yml index 9cfd8bd74b1..5a70868ed6e 100644 --- a/.github/workflows/artifacts-summary.lock.yml +++ b/.github/workflows/artifacts-summary.lock.yml @@ -1094,185 +1094,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/audit-workflows.lock.yml b/.github/workflows/audit-workflows.lock.yml index 91ef95d2ced..c28e40330a1 100644 --- a/.github/workflows/audit-workflows.lock.yml +++ b/.github/workflows/audit-workflows.lock.yml @@ -1388,185 +1388,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/auto-triage-issues.lock.yml b/.github/workflows/auto-triage-issues.lock.yml index 2f577c402bb..1c01324078a 100644 --- a/.github/workflows/auto-triage-issues.lock.yml +++ b/.github/workflows/auto-triage-issues.lock.yml @@ -1121,185 +1121,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/avenger.lock.yml b/.github/workflows/avenger.lock.yml index acc63d9d075..e58b415a3ad 100644 --- a/.github/workflows/avenger.lock.yml +++ b/.github/workflows/avenger.lock.yml @@ -1278,185 +1278,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/aw-failure-investigator.lock.yml b/.github/workflows/aw-failure-investigator.lock.yml index ffe0666ebd0..4db51a98c28 100644 --- a/.github/workflows/aw-failure-investigator.lock.yml +++ b/.github/workflows/aw-failure-investigator.lock.yml @@ -1379,185 +1379,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/blog-auditor.lock.yml b/.github/workflows/blog-auditor.lock.yml index 04185b47dba..07299d5cb54 100644 --- a/.github/workflows/blog-auditor.lock.yml +++ b/.github/workflows/blog-auditor.lock.yml @@ -1257,185 +1257,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/bot-detection.lock.yml b/.github/workflows/bot-detection.lock.yml index 0baa7fb6b88..289511cde38 100644 --- a/.github/workflows/bot-detection.lock.yml +++ b/.github/workflows/bot-detection.lock.yml @@ -1182,185 +1182,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/brave.lock.yml b/.github/workflows/brave.lock.yml index 18f1ad656e5..8e1563d6301 100644 --- a/.github/workflows/brave.lock.yml +++ b/.github/workflows/brave.lock.yml @@ -1176,185 +1176,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/breaking-change-checker.lock.yml b/.github/workflows/breaking-change-checker.lock.yml index 63636ece599..b86aa1a9783 100644 --- a/.github/workflows/breaking-change-checker.lock.yml +++ b/.github/workflows/breaking-change-checker.lock.yml @@ -1136,185 +1136,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/changeset.lock.yml b/.github/workflows/changeset.lock.yml index ec0bb45707e..2416a9cea17 100644 --- a/.github/workflows/changeset.lock.yml +++ b/.github/workflows/changeset.lock.yml @@ -1223,185 +1223,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/chaos-pr-bundle-fuzzer.lock.yml b/.github/workflows/chaos-pr-bundle-fuzzer.lock.yml index 2a645372401..ec511be3d39 100644 --- a/.github/workflows/chaos-pr-bundle-fuzzer.lock.yml +++ b/.github/workflows/chaos-pr-bundle-fuzzer.lock.yml @@ -1120,185 +1120,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/ci-coach.lock.yml b/.github/workflows/ci-coach.lock.yml index 6d441b5cbca..fda4186d936 100644 --- a/.github/workflows/ci-coach.lock.yml +++ b/.github/workflows/ci-coach.lock.yml @@ -1230,185 +1230,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/ci-doctor.lock.yml b/.github/workflows/ci-doctor.lock.yml index d67f0a5f4a1..09e263eda5b 100644 --- a/.github/workflows/ci-doctor.lock.yml +++ b/.github/workflows/ci-doctor.lock.yml @@ -1399,185 +1399,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/claude-code-user-docs-review.lock.yml b/.github/workflows/claude-code-user-docs-review.lock.yml index b07f898a232..c29fb83ff20 100644 --- a/.github/workflows/claude-code-user-docs-review.lock.yml +++ b/.github/workflows/claude-code-user-docs-review.lock.yml @@ -1225,185 +1225,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/cli-consistency-checker.lock.yml b/.github/workflows/cli-consistency-checker.lock.yml index 3a4b4722f23..4a68d6749db 100644 --- a/.github/workflows/cli-consistency-checker.lock.yml +++ b/.github/workflows/cli-consistency-checker.lock.yml @@ -1109,185 +1109,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/cli-version-checker.lock.yml b/.github/workflows/cli-version-checker.lock.yml index d9d1838f43e..9c476c82507 100644 --- a/.github/workflows/cli-version-checker.lock.yml +++ b/.github/workflows/cli-version-checker.lock.yml @@ -1219,185 +1219,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/cloclo.lock.yml b/.github/workflows/cloclo.lock.yml index 87f510a0290..03b4f6373d0 100644 --- a/.github/workflows/cloclo.lock.yml +++ b/.github/workflows/cloclo.lock.yml @@ -1506,185 +1506,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/code-scanning-fixer.lock.yml b/.github/workflows/code-scanning-fixer.lock.yml index 526345345db..077bc0702a3 100644 --- a/.github/workflows/code-scanning-fixer.lock.yml +++ b/.github/workflows/code-scanning-fixer.lock.yml @@ -1217,185 +1217,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/code-simplifier.lock.yml b/.github/workflows/code-simplifier.lock.yml index 141f38c40c0..adf1e231d18 100644 --- a/.github/workflows/code-simplifier.lock.yml +++ b/.github/workflows/code-simplifier.lock.yml @@ -1173,185 +1173,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/codex-github-remote-mcp-test.lock.yml b/.github/workflows/codex-github-remote-mcp-test.lock.yml index 4ab33e0877f..4319c2fbd87 100644 --- a/.github/workflows/codex-github-remote-mcp-test.lock.yml +++ b/.github/workflows/codex-github-remote-mcp-test.lock.yml @@ -1057,185 +1057,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/commit-changes-analyzer.lock.yml b/.github/workflows/commit-changes-analyzer.lock.yml index d63c3a709d5..632491d9691 100644 --- a/.github/workflows/commit-changes-analyzer.lock.yml +++ b/.github/workflows/commit-changes-analyzer.lock.yml @@ -1065,185 +1065,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/constraint-solving-potd.lock.yml b/.github/workflows/constraint-solving-potd.lock.yml index b76fdece349..80518140b5b 100644 --- a/.github/workflows/constraint-solving-potd.lock.yml +++ b/.github/workflows/constraint-solving-potd.lock.yml @@ -1119,185 +1119,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/contribution-check.lock.yml b/.github/workflows/contribution-check.lock.yml index 098cf97a357..c376260555d 100644 --- a/.github/workflows/contribution-check.lock.yml +++ b/.github/workflows/contribution-check.lock.yml @@ -1232,185 +1232,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/copilot-agent-analysis.lock.yml b/.github/workflows/copilot-agent-analysis.lock.yml index 3be5db6bc52..26c8e3df13f 100644 --- a/.github/workflows/copilot-agent-analysis.lock.yml +++ b/.github/workflows/copilot-agent-analysis.lock.yml @@ -1302,185 +1302,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/copilot-centralization-drilldown.lock.yml b/.github/workflows/copilot-centralization-drilldown.lock.yml index aadac057bdb..587a1b9fac3 100644 --- a/.github/workflows/copilot-centralization-drilldown.lock.yml +++ b/.github/workflows/copilot-centralization-drilldown.lock.yml @@ -1082,185 +1082,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/copilot-centralization-optimizer.lock.yml b/.github/workflows/copilot-centralization-optimizer.lock.yml index 6f4ce489b8f..74c7d868ad3 100644 --- a/.github/workflows/copilot-centralization-optimizer.lock.yml +++ b/.github/workflows/copilot-centralization-optimizer.lock.yml @@ -1130,185 +1130,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/copilot-cli-deep-research.lock.yml b/.github/workflows/copilot-cli-deep-research.lock.yml index 68a7e28de60..79bebb1a4c3 100644 --- a/.github/workflows/copilot-cli-deep-research.lock.yml +++ b/.github/workflows/copilot-cli-deep-research.lock.yml @@ -1137,185 +1137,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/copilot-opt.lock.yml b/.github/workflows/copilot-opt.lock.yml index fd66c073e4e..ddc5818047b 100644 --- a/.github/workflows/copilot-opt.lock.yml +++ b/.github/workflows/copilot-opt.lock.yml @@ -1207,185 +1207,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/copilot-pr-merged-report.lock.yml b/.github/workflows/copilot-pr-merged-report.lock.yml index 7b68cd299f8..002d92818d1 100644 --- a/.github/workflows/copilot-pr-merged-report.lock.yml +++ b/.github/workflows/copilot-pr-merged-report.lock.yml @@ -1075,185 +1075,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/copilot-pr-nlp-analysis.lock.yml b/.github/workflows/copilot-pr-nlp-analysis.lock.yml index 94435c4165e..a8d1a9a0ffa 100644 --- a/.github/workflows/copilot-pr-nlp-analysis.lock.yml +++ b/.github/workflows/copilot-pr-nlp-analysis.lock.yml @@ -1263,185 +1263,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/copilot-pr-prompt-analysis.lock.yml b/.github/workflows/copilot-pr-prompt-analysis.lock.yml index 30777759f7a..15d3fe50c92 100644 --- a/.github/workflows/copilot-pr-prompt-analysis.lock.yml +++ b/.github/workflows/copilot-pr-prompt-analysis.lock.yml @@ -1202,185 +1202,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/copilot-session-insights.lock.yml b/.github/workflows/copilot-session-insights.lock.yml index 7a4b58147bd..5293e78ae72 100644 --- a/.github/workflows/copilot-session-insights.lock.yml +++ b/.github/workflows/copilot-session-insights.lock.yml @@ -1321,185 +1321,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/craft.lock.yml b/.github/workflows/craft.lock.yml index 54d1bbbaa4e..94fd2d9d518 100644 --- a/.github/workflows/craft.lock.yml +++ b/.github/workflows/craft.lock.yml @@ -1177,185 +1177,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-agent-of-the-day-blog-writer.lock.yml b/.github/workflows/daily-agent-of-the-day-blog-writer.lock.yml index 9c249defddf..1c007e06d37 100644 --- a/.github/workflows/daily-agent-of-the-day-blog-writer.lock.yml +++ b/.github/workflows/daily-agent-of-the-day-blog-writer.lock.yml @@ -1310,185 +1310,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-agentrx-trace-optimizer.lock.yml b/.github/workflows/daily-agentrx-trace-optimizer.lock.yml index 1bcfe1756b0..02d5eb8ed99 100644 --- a/.github/workflows/daily-agentrx-trace-optimizer.lock.yml +++ b/.github/workflows/daily-agentrx-trace-optimizer.lock.yml @@ -1326,185 +1326,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-ambient-context-optimizer.lock.yml b/.github/workflows/daily-ambient-context-optimizer.lock.yml index fd24e4a847f..faa32e71dd4 100644 --- a/.github/workflows/daily-ambient-context-optimizer.lock.yml +++ b/.github/workflows/daily-ambient-context-optimizer.lock.yml @@ -1195,185 +1195,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-architecture-diagram.lock.yml b/.github/workflows/daily-architecture-diagram.lock.yml index f75d6112df7..7342a04f47d 100644 --- a/.github/workflows/daily-architecture-diagram.lock.yml +++ b/.github/workflows/daily-architecture-diagram.lock.yml @@ -1267,185 +1267,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-assign-issue-to-user.lock.yml b/.github/workflows/daily-assign-issue-to-user.lock.yml index 7cbd53febdc..d9911a3174b 100644 --- a/.github/workflows/daily-assign-issue-to-user.lock.yml +++ b/.github/workflows/daily-assign-issue-to-user.lock.yml @@ -1106,185 +1106,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-astrostylelite-markdown-spellcheck.lock.yml b/.github/workflows/daily-astrostylelite-markdown-spellcheck.lock.yml index eb15157ebe1..ca883aad984 100644 --- a/.github/workflows/daily-astrostylelite-markdown-spellcheck.lock.yml +++ b/.github/workflows/daily-astrostylelite-markdown-spellcheck.lock.yml @@ -1220,185 +1220,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-aw-cross-repo-compile-check.lock.yml b/.github/workflows/daily-aw-cross-repo-compile-check.lock.yml index 6e46be7c795..0d4447668f8 100644 --- a/.github/workflows/daily-aw-cross-repo-compile-check.lock.yml +++ b/.github/workflows/daily-aw-cross-repo-compile-check.lock.yml @@ -1215,185 +1215,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-awf-spec-compiler-surfacing.lock.yml b/.github/workflows/daily-awf-spec-compiler-surfacing.lock.yml index 7570890be11..d893c4bb9f4 100644 --- a/.github/workflows/daily-awf-spec-compiler-surfacing.lock.yml +++ b/.github/workflows/daily-awf-spec-compiler-surfacing.lock.yml @@ -1108,185 +1108,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-byok-ollama-test.lock.yml b/.github/workflows/daily-byok-ollama-test.lock.yml index 059803637e6..a492bf3ecce 100644 --- a/.github/workflows/daily-byok-ollama-test.lock.yml +++ b/.github/workflows/daily-byok-ollama-test.lock.yml @@ -1085,185 +1085,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-cache-strategy-analyzer.lock.yml b/.github/workflows/daily-cache-strategy-analyzer.lock.yml index 313474d98dd..9c55ec84cd7 100644 --- a/.github/workflows/daily-cache-strategy-analyzer.lock.yml +++ b/.github/workflows/daily-cache-strategy-analyzer.lock.yml @@ -1353,185 +1353,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-caveman-optimizer.lock.yml b/.github/workflows/daily-caveman-optimizer.lock.yml index 910eaf2b7a1..39fadac4b2d 100644 --- a/.github/workflows/daily-caveman-optimizer.lock.yml +++ b/.github/workflows/daily-caveman-optimizer.lock.yml @@ -1258,185 +1258,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-choice-test.lock.yml b/.github/workflows/daily-choice-test.lock.yml index fbec61e5377..3679a6b22be 100644 --- a/.github/workflows/daily-choice-test.lock.yml +++ b/.github/workflows/daily-choice-test.lock.yml @@ -1153,185 +1153,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-cli-performance.lock.yml b/.github/workflows/daily-cli-performance.lock.yml index 15a0865b509..cbb8bf33d8c 100644 --- a/.github/workflows/daily-cli-performance.lock.yml +++ b/.github/workflows/daily-cli-performance.lock.yml @@ -1389,185 +1389,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-cli-tools-tester.lock.yml b/.github/workflows/daily-cli-tools-tester.lock.yml index 846de22b381..c97b52d1c03 100644 --- a/.github/workflows/daily-cli-tools-tester.lock.yml +++ b/.github/workflows/daily-cli-tools-tester.lock.yml @@ -1220,185 +1220,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-code-metrics.lock.yml b/.github/workflows/daily-code-metrics.lock.yml index 72fc0ed6e57..233f1347e8a 100644 --- a/.github/workflows/daily-code-metrics.lock.yml +++ b/.github/workflows/daily-code-metrics.lock.yml @@ -1339,185 +1339,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-community-attribution.lock.yml b/.github/workflows/daily-community-attribution.lock.yml index 209c78b65a7..d1c67b10b52 100644 --- a/.github/workflows/daily-community-attribution.lock.yml +++ b/.github/workflows/daily-community-attribution.lock.yml @@ -1279,185 +1279,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-compiler-quality.lock.yml b/.github/workflows/daily-compiler-quality.lock.yml index 82165088496..029d0ebba52 100644 --- a/.github/workflows/daily-compiler-quality.lock.yml +++ b/.github/workflows/daily-compiler-quality.lock.yml @@ -1255,185 +1255,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-compiler-threat-spec-optimizer.lock.yml b/.github/workflows/daily-compiler-threat-spec-optimizer.lock.yml index 68cc8de18b9..3ca5fe7acfe 100644 --- a/.github/workflows/daily-compiler-threat-spec-optimizer.lock.yml +++ b/.github/workflows/daily-compiler-threat-spec-optimizer.lock.yml @@ -1180,185 +1180,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-credit-limit-test.lock.yml b/.github/workflows/daily-credit-limit-test.lock.yml index ae4fe1d5e8a..ed6bc516ee0 100644 --- a/.github/workflows/daily-credit-limit-test.lock.yml +++ b/.github/workflows/daily-credit-limit-test.lock.yml @@ -1063,185 +1063,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-doc-healer.lock.yml b/.github/workflows/daily-doc-healer.lock.yml index d29218973a1..bc471bdc35f 100644 --- a/.github/workflows/daily-doc-healer.lock.yml +++ b/.github/workflows/daily-doc-healer.lock.yml @@ -1362,185 +1362,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-doc-updater.lock.yml b/.github/workflows/daily-doc-updater.lock.yml index b2bd742b4e8..924c17baa89 100644 --- a/.github/workflows/daily-doc-updater.lock.yml +++ b/.github/workflows/daily-doc-updater.lock.yml @@ -1164,185 +1164,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-experiment-report.lock.yml b/.github/workflows/daily-experiment-report.lock.yml index 31fd113c906..2c4722aeb45 100644 --- a/.github/workflows/daily-experiment-report.lock.yml +++ b/.github/workflows/daily-experiment-report.lock.yml @@ -1251,185 +1251,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-fact.lock.yml b/.github/workflows/daily-fact.lock.yml index 6d1acb2e402..962a8afa084 100644 --- a/.github/workflows/daily-fact.lock.yml +++ b/.github/workflows/daily-fact.lock.yml @@ -1366,185 +1366,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-file-diet.lock.yml b/.github/workflows/daily-file-diet.lock.yml index a5d446b7751..4870b577c95 100644 --- a/.github/workflows/daily-file-diet.lock.yml +++ b/.github/workflows/daily-file-diet.lock.yml @@ -1177,185 +1177,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-firewall-report.lock.yml b/.github/workflows/daily-firewall-report.lock.yml index bf5be9c7d91..e17dd52414c 100644 --- a/.github/workflows/daily-firewall-report.lock.yml +++ b/.github/workflows/daily-firewall-report.lock.yml @@ -1179,185 +1179,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-formal-spec-verifier.lock.yml b/.github/workflows/daily-formal-spec-verifier.lock.yml index 3c0d10ad518..b02a125198d 100644 --- a/.github/workflows/daily-formal-spec-verifier.lock.yml +++ b/.github/workflows/daily-formal-spec-verifier.lock.yml @@ -1223,185 +1223,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-function-namer.lock.yml b/.github/workflows/daily-function-namer.lock.yml index 18a321e8e3c..5e0163cdc27 100644 --- a/.github/workflows/daily-function-namer.lock.yml +++ b/.github/workflows/daily-function-namer.lock.yml @@ -1180,185 +1180,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-geo-optimizer.lock.yml b/.github/workflows/daily-geo-optimizer.lock.yml index 65ebec557f3..231ed5670e8 100644 --- a/.github/workflows/daily-geo-optimizer.lock.yml +++ b/.github/workflows/daily-geo-optimizer.lock.yml @@ -1131,185 +1131,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-hippo-learn.lock.yml b/.github/workflows/daily-hippo-learn.lock.yml index 59999d8bf94..e0c435dbfc6 100644 --- a/.github/workflows/daily-hippo-learn.lock.yml +++ b/.github/workflows/daily-hippo-learn.lock.yml @@ -1234,185 +1234,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-issues-report.lock.yml b/.github/workflows/daily-issues-report.lock.yml index a0e65f7faac..2d9b888e489 100644 --- a/.github/workflows/daily-issues-report.lock.yml +++ b/.github/workflows/daily-issues-report.lock.yml @@ -1406,185 +1406,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-malicious-code-scan.lock.yml b/.github/workflows/daily-malicious-code-scan.lock.yml index ab71b41cb3f..c9df5951e9c 100644 --- a/.github/workflows/daily-malicious-code-scan.lock.yml +++ b/.github/workflows/daily-malicious-code-scan.lock.yml @@ -1141,185 +1141,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-max-ai-credits-test.lock.yml b/.github/workflows/daily-max-ai-credits-test.lock.yml index 81968b10efc..87f9d803a1d 100644 --- a/.github/workflows/daily-max-ai-credits-test.lock.yml +++ b/.github/workflows/daily-max-ai-credits-test.lock.yml @@ -1002,185 +1002,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-mcp-concurrency-analysis.lock.yml b/.github/workflows/daily-mcp-concurrency-analysis.lock.yml index c8896b60a9c..8eb7d8d4a9b 100644 --- a/.github/workflows/daily-mcp-concurrency-analysis.lock.yml +++ b/.github/workflows/daily-mcp-concurrency-analysis.lock.yml @@ -1259,185 +1259,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-model-inventory.lock.yml b/.github/workflows/daily-model-inventory.lock.yml index 6ace2d03c98..2e2df5ccdb1 100644 --- a/.github/workflows/daily-model-inventory.lock.yml +++ b/.github/workflows/daily-model-inventory.lock.yml @@ -1449,185 +1449,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-multi-device-docs-tester.lock.yml b/.github/workflows/daily-multi-device-docs-tester.lock.yml index 85544e1cbb3..4235008ae8a 100644 --- a/.github/workflows/daily-multi-device-docs-tester.lock.yml +++ b/.github/workflows/daily-multi-device-docs-tester.lock.yml @@ -1154,185 +1154,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-news.lock.yml b/.github/workflows/daily-news.lock.yml index 043f585aaa5..f5b8600f798 100644 --- a/.github/workflows/daily-news.lock.yml +++ b/.github/workflows/daily-news.lock.yml @@ -1374,185 +1374,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-observability-report.lock.yml b/.github/workflows/daily-observability-report.lock.yml index 63296423eca..c228b96e461 100644 --- a/.github/workflows/daily-observability-report.lock.yml +++ b/.github/workflows/daily-observability-report.lock.yml @@ -1225,185 +1225,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-performance-summary.lock.yml b/.github/workflows/daily-performance-summary.lock.yml index a709d96d589..c0a39be5608 100644 --- a/.github/workflows/daily-performance-summary.lock.yml +++ b/.github/workflows/daily-performance-summary.lock.yml @@ -1689,185 +1689,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-regulatory.lock.yml b/.github/workflows/daily-regulatory.lock.yml index 1bfefa330a6..dc5d9c4f75b 100644 --- a/.github/workflows/daily-regulatory.lock.yml +++ b/.github/workflows/daily-regulatory.lock.yml @@ -1618,185 +1618,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-reliability-review.lock.yml b/.github/workflows/daily-reliability-review.lock.yml index 9da3ae481d3..937cc191d11 100644 --- a/.github/workflows/daily-reliability-review.lock.yml +++ b/.github/workflows/daily-reliability-review.lock.yml @@ -1237,185 +1237,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-rendering-scripts-verifier.lock.yml b/.github/workflows/daily-rendering-scripts-verifier.lock.yml index 449ab74ee5c..cc82317938c 100644 --- a/.github/workflows/daily-rendering-scripts-verifier.lock.yml +++ b/.github/workflows/daily-rendering-scripts-verifier.lock.yml @@ -1390,185 +1390,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-repo-chronicle.lock.yml b/.github/workflows/daily-repo-chronicle.lock.yml index d1c9a07c9df..4f24375d9ae 100644 --- a/.github/workflows/daily-repo-chronicle.lock.yml +++ b/.github/workflows/daily-repo-chronicle.lock.yml @@ -1195,185 +1195,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-safe-output-integrator.lock.yml b/.github/workflows/daily-safe-output-integrator.lock.yml index 7876aa78a75..8d87d349713 100644 --- a/.github/workflows/daily-safe-output-integrator.lock.yml +++ b/.github/workflows/daily-safe-output-integrator.lock.yml @@ -1179,185 +1179,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-safe-output-optimizer.lock.yml b/.github/workflows/daily-safe-output-optimizer.lock.yml index 0aab8f90d87..a94cdd8da3b 100644 --- a/.github/workflows/daily-safe-output-optimizer.lock.yml +++ b/.github/workflows/daily-safe-output-optimizer.lock.yml @@ -1411,185 +1411,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-safe-outputs-conformance.lock.yml b/.github/workflows/daily-safe-outputs-conformance.lock.yml index f33761f2ef5..0096d3e1da4 100644 --- a/.github/workflows/daily-safe-outputs-conformance.lock.yml +++ b/.github/workflows/daily-safe-outputs-conformance.lock.yml @@ -1193,185 +1193,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-safeoutputs-git-simulator.lock.yml b/.github/workflows/daily-safeoutputs-git-simulator.lock.yml index e2e25be8a58..64ea28d036e 100644 --- a/.github/workflows/daily-safeoutputs-git-simulator.lock.yml +++ b/.github/workflows/daily-safeoutputs-git-simulator.lock.yml @@ -1252,185 +1252,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-secrets-analysis.lock.yml b/.github/workflows/daily-secrets-analysis.lock.yml index d5571965124..ff1fa9dc792 100644 --- a/.github/workflows/daily-secrets-analysis.lock.yml +++ b/.github/workflows/daily-secrets-analysis.lock.yml @@ -1097,185 +1097,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-security-observability.lock.yml b/.github/workflows/daily-security-observability.lock.yml index 7d06e9ffa46..4e38728db29 100644 --- a/.github/workflows/daily-security-observability.lock.yml +++ b/.github/workflows/daily-security-observability.lock.yml @@ -1321,185 +1321,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-security-red-team.lock.yml b/.github/workflows/daily-security-red-team.lock.yml index dc13f8006a2..aa5818e46e1 100644 --- a/.github/workflows/daily-security-red-team.lock.yml +++ b/.github/workflows/daily-security-red-team.lock.yml @@ -1290,185 +1290,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-semgrep-scan.lock.yml b/.github/workflows/daily-semgrep-scan.lock.yml index 4f5d135c764..a1d6dff91f7 100644 --- a/.github/workflows/daily-semgrep-scan.lock.yml +++ b/.github/workflows/daily-semgrep-scan.lock.yml @@ -1177,185 +1177,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-sentrux-report.lock.yml b/.github/workflows/daily-sentrux-report.lock.yml index dadbbad84b3..b2d11f4f50b 100644 --- a/.github/workflows/daily-sentrux-report.lock.yml +++ b/.github/workflows/daily-sentrux-report.lock.yml @@ -1154,185 +1154,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-skill-optimizer.lock.yml b/.github/workflows/daily-skill-optimizer.lock.yml index 197a0070441..1e8b40b6885 100644 --- a/.github/workflows/daily-skill-optimizer.lock.yml +++ b/.github/workflows/daily-skill-optimizer.lock.yml @@ -1120,185 +1120,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-spdd-spec-planner.lock.yml b/.github/workflows/daily-spdd-spec-planner.lock.yml index 7c640a94b50..301aa1a84e7 100644 --- a/.github/workflows/daily-spdd-spec-planner.lock.yml +++ b/.github/workflows/daily-spdd-spec-planner.lock.yml @@ -1182,185 +1182,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-syntax-error-quality.lock.yml b/.github/workflows/daily-syntax-error-quality.lock.yml index 0afa3c72759..b5034838f16 100644 --- a/.github/workflows/daily-syntax-error-quality.lock.yml +++ b/.github/workflows/daily-syntax-error-quality.lock.yml @@ -1121,185 +1121,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-team-evolution-insights.lock.yml b/.github/workflows/daily-team-evolution-insights.lock.yml index 42c987f78bc..8feb9397e0d 100644 --- a/.github/workflows/daily-team-evolution-insights.lock.yml +++ b/.github/workflows/daily-team-evolution-insights.lock.yml @@ -1162,185 +1162,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-team-status.lock.yml b/.github/workflows/daily-team-status.lock.yml index 8743f89fd0a..aa74e2c5c07 100644 --- a/.github/workflows/daily-team-status.lock.yml +++ b/.github/workflows/daily-team-status.lock.yml @@ -1078,185 +1078,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-testify-uber-super-expert.lock.yml b/.github/workflows/daily-testify-uber-super-expert.lock.yml index 82f0c3fb602..0e5b8e4ead4 100644 --- a/.github/workflows/daily-testify-uber-super-expert.lock.yml +++ b/.github/workflows/daily-testify-uber-super-expert.lock.yml @@ -1226,185 +1226,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-token-consumption-report.lock.yml b/.github/workflows/daily-token-consumption-report.lock.yml index 0af77359882..7ec8d776b7f 100644 --- a/.github/workflows/daily-token-consumption-report.lock.yml +++ b/.github/workflows/daily-token-consumption-report.lock.yml @@ -1316,185 +1316,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-windows-terminal-integration-builder.lock.yml b/.github/workflows/daily-windows-terminal-integration-builder.lock.yml index 40a18065ea4..fc5510ce7a8 100644 --- a/.github/workflows/daily-windows-terminal-integration-builder.lock.yml +++ b/.github/workflows/daily-windows-terminal-integration-builder.lock.yml @@ -1061,185 +1061,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/daily-workflow-updater.lock.yml b/.github/workflows/daily-workflow-updater.lock.yml index 22cb0756692..37d61ce59dd 100644 --- a/.github/workflows/daily-workflow-updater.lock.yml +++ b/.github/workflows/daily-workflow-updater.lock.yml @@ -1108,185 +1108,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/dataflow-pr-discussion-dataset.lock.yml b/.github/workflows/dataflow-pr-discussion-dataset.lock.yml index 582a8cc423b..c81e2ae9632 100644 --- a/.github/workflows/dataflow-pr-discussion-dataset.lock.yml +++ b/.github/workflows/dataflow-pr-discussion-dataset.lock.yml @@ -1470,185 +1470,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/dead-code-remover.lock.yml b/.github/workflows/dead-code-remover.lock.yml index 6d6ca9ba5df..e9212353795 100644 --- a/.github/workflows/dead-code-remover.lock.yml +++ b/.github/workflows/dead-code-remover.lock.yml @@ -1180,185 +1180,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/deep-report.lock.yml b/.github/workflows/deep-report.lock.yml index fe0fc015317..821747cfbbb 100644 --- a/.github/workflows/deep-report.lock.yml +++ b/.github/workflows/deep-report.lock.yml @@ -1662,185 +1662,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/delight.lock.yml b/.github/workflows/delight.lock.yml index 656b99509b1..658040a4c22 100644 --- a/.github/workflows/delight.lock.yml +++ b/.github/workflows/delight.lock.yml @@ -1209,185 +1209,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/dependabot-burner.lock.yml b/.github/workflows/dependabot-burner.lock.yml index e028b755f84..30b13ec625c 100644 --- a/.github/workflows/dependabot-burner.lock.yml +++ b/.github/workflows/dependabot-burner.lock.yml @@ -1252,185 +1252,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/dependabot-go-checker.lock.yml b/.github/workflows/dependabot-go-checker.lock.yml index 9ca922af087..dbb9ca24388 100644 --- a/.github/workflows/dependabot-go-checker.lock.yml +++ b/.github/workflows/dependabot-go-checker.lock.yml @@ -1168,185 +1168,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/dependabot-repair.lock.yml b/.github/workflows/dependabot-repair.lock.yml index d286b8e9d0a..8411e799287 100644 --- a/.github/workflows/dependabot-repair.lock.yml +++ b/.github/workflows/dependabot-repair.lock.yml @@ -1208,185 +1208,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/deployment-incident-monitor.lock.yml b/.github/workflows/deployment-incident-monitor.lock.yml index 49d52b89571..36d6f5c79f8 100644 --- a/.github/workflows/deployment-incident-monitor.lock.yml +++ b/.github/workflows/deployment-incident-monitor.lock.yml @@ -1118,185 +1118,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/design-decision-gate.lock.yml b/.github/workflows/design-decision-gate.lock.yml index eb68b658c88..2eb30fc9859 100644 --- a/.github/workflows/design-decision-gate.lock.yml +++ b/.github/workflows/design-decision-gate.lock.yml @@ -1303,185 +1303,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/designer-drift-audit.lock.yml b/.github/workflows/designer-drift-audit.lock.yml index 327f7945190..e57b147e896 100644 --- a/.github/workflows/designer-drift-audit.lock.yml +++ b/.github/workflows/designer-drift-audit.lock.yml @@ -1067,185 +1067,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/dev-hawk.lock.yml b/.github/workflows/dev-hawk.lock.yml index 039bfdd4467..af344a83abb 100644 --- a/.github/workflows/dev-hawk.lock.yml +++ b/.github/workflows/dev-hawk.lock.yml @@ -1226,185 +1226,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/dev.lock.yml b/.github/workflows/dev.lock.yml index 023cb6f515f..85e65d3c5e6 100644 --- a/.github/workflows/dev.lock.yml +++ b/.github/workflows/dev.lock.yml @@ -1186,185 +1186,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/developer-docs-consolidator.lock.yml b/.github/workflows/developer-docs-consolidator.lock.yml index 545dd5b2d1e..4254ab45e06 100644 --- a/.github/workflows/developer-docs-consolidator.lock.yml +++ b/.github/workflows/developer-docs-consolidator.lock.yml @@ -1359,185 +1359,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/dictation-prompt.lock.yml b/.github/workflows/dictation-prompt.lock.yml index 673cf77827c..8d439c9547e 100644 --- a/.github/workflows/dictation-prompt.lock.yml +++ b/.github/workflows/dictation-prompt.lock.yml @@ -1110,185 +1110,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/discussion-task-miner.lock.yml b/.github/workflows/discussion-task-miner.lock.yml index b4f0710ed98..694e22ef262 100644 --- a/.github/workflows/discussion-task-miner.lock.yml +++ b/.github/workflows/discussion-task-miner.lock.yml @@ -1192,185 +1192,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/docs-noob-tester.lock.yml b/.github/workflows/docs-noob-tester.lock.yml index 2a195b445f4..bba764c5831 100644 --- a/.github/workflows/docs-noob-tester.lock.yml +++ b/.github/workflows/docs-noob-tester.lock.yml @@ -1162,185 +1162,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/draft-pr-cleanup.lock.yml b/.github/workflows/draft-pr-cleanup.lock.yml index 7531a4327ba..f37f8bf2d0e 100644 --- a/.github/workflows/draft-pr-cleanup.lock.yml +++ b/.github/workflows/draft-pr-cleanup.lock.yml @@ -1144,185 +1144,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/duplicate-code-detector.lock.yml b/.github/workflows/duplicate-code-detector.lock.yml index 3b07fe06f73..d5342aea91a 100644 --- a/.github/workflows/duplicate-code-detector.lock.yml +++ b/.github/workflows/duplicate-code-detector.lock.yml @@ -1203,185 +1203,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/example-failure-category-filter.lock.yml b/.github/workflows/example-failure-category-filter.lock.yml index 8983ead27b9..993e3db5579 100644 --- a/.github/workflows/example-failure-category-filter.lock.yml +++ b/.github/workflows/example-failure-category-filter.lock.yml @@ -1055,185 +1055,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/example-permissions-warning.lock.yml b/.github/workflows/example-permissions-warning.lock.yml index 7e3f040dd5f..7aee6c22b72 100644 --- a/.github/workflows/example-permissions-warning.lock.yml +++ b/.github/workflows/example-permissions-warning.lock.yml @@ -1018,185 +1018,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/example-workflow-analyzer.lock.yml b/.github/workflows/example-workflow-analyzer.lock.yml index f4c1aeff624..a4656abd74d 100644 --- a/.github/workflows/example-workflow-analyzer.lock.yml +++ b/.github/workflows/example-workflow-analyzer.lock.yml @@ -1242,185 +1242,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/firewall-escape.lock.yml b/.github/workflows/firewall-escape.lock.yml index 2f01e1f7548..c9a92e0bb84 100644 --- a/.github/workflows/firewall-escape.lock.yml +++ b/.github/workflows/firewall-escape.lock.yml @@ -1203,185 +1203,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/firewall.lock.yml b/.github/workflows/firewall.lock.yml index c324621094c..8472d48d8cf 100644 --- a/.github/workflows/firewall.lock.yml +++ b/.github/workflows/firewall.lock.yml @@ -1026,185 +1026,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/functional-pragmatist.lock.yml b/.github/workflows/functional-pragmatist.lock.yml index 56175a98bb0..9a284b4f4b7 100644 --- a/.github/workflows/functional-pragmatist.lock.yml +++ b/.github/workflows/functional-pragmatist.lock.yml @@ -1116,185 +1116,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/github-mcp-structural-analysis.lock.yml b/.github/workflows/github-mcp-structural-analysis.lock.yml index fb168673a00..5c53a86d6b1 100644 --- a/.github/workflows/github-mcp-structural-analysis.lock.yml +++ b/.github/workflows/github-mcp-structural-analysis.lock.yml @@ -1265,185 +1265,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/github-mcp-tools-report.lock.yml b/.github/workflows/github-mcp-tools-report.lock.yml index aebaaa82620..720065d32df 100644 --- a/.github/workflows/github-mcp-tools-report.lock.yml +++ b/.github/workflows/github-mcp-tools-report.lock.yml @@ -1256,185 +1256,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/github-remote-mcp-auth-test.lock.yml b/.github/workflows/github-remote-mcp-auth-test.lock.yml index 7c04eb1f8bc..49277ff7f2f 100644 --- a/.github/workflows/github-remote-mcp-auth-test.lock.yml +++ b/.github/workflows/github-remote-mcp-auth-test.lock.yml @@ -1112,185 +1112,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/glossary-maintainer.lock.yml b/.github/workflows/glossary-maintainer.lock.yml index 9704caf9711..3b29598b6b7 100644 --- a/.github/workflows/glossary-maintainer.lock.yml +++ b/.github/workflows/glossary-maintainer.lock.yml @@ -1260,185 +1260,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/go-fan.lock.yml b/.github/workflows/go-fan.lock.yml index c0b1f0ce098..96960301fca 100644 --- a/.github/workflows/go-fan.lock.yml +++ b/.github/workflows/go-fan.lock.yml @@ -1288,185 +1288,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/go-logger.lock.yml b/.github/workflows/go-logger.lock.yml index b97f9db4999..ced694016c1 100644 --- a/.github/workflows/go-logger.lock.yml +++ b/.github/workflows/go-logger.lock.yml @@ -1272,185 +1272,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/go-pattern-detector.lock.yml b/.github/workflows/go-pattern-detector.lock.yml index 2b90f7d10d5..821680db0d3 100644 --- a/.github/workflows/go-pattern-detector.lock.yml +++ b/.github/workflows/go-pattern-detector.lock.yml @@ -1236,185 +1236,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/gpclean.lock.yml b/.github/workflows/gpclean.lock.yml index f66b7462078..8ef90b62216 100644 --- a/.github/workflows/gpclean.lock.yml +++ b/.github/workflows/gpclean.lock.yml @@ -1197,185 +1197,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/grumpy-reviewer.lock.yml b/.github/workflows/grumpy-reviewer.lock.yml index d4cdee114d6..0273eb8f45d 100644 --- a/.github/workflows/grumpy-reviewer.lock.yml +++ b/.github/workflows/grumpy-reviewer.lock.yml @@ -1238,185 +1238,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/hippo-embed.lock.yml b/.github/workflows/hippo-embed.lock.yml index 55f27fc54db..9254e963594 100644 --- a/.github/workflows/hippo-embed.lock.yml +++ b/.github/workflows/hippo-embed.lock.yml @@ -1148,185 +1148,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/hourly-ci-cleaner.lock.yml b/.github/workflows/hourly-ci-cleaner.lock.yml index 8bc2573c0a6..927947b8f7f 100644 --- a/.github/workflows/hourly-ci-cleaner.lock.yml +++ b/.github/workflows/hourly-ci-cleaner.lock.yml @@ -1273,185 +1273,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/instructions-janitor.lock.yml b/.github/workflows/instructions-janitor.lock.yml index e9a9e5869e7..cbf91324d68 100644 --- a/.github/workflows/instructions-janitor.lock.yml +++ b/.github/workflows/instructions-janitor.lock.yml @@ -1247,185 +1247,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/issue-arborist.lock.yml b/.github/workflows/issue-arborist.lock.yml index ef5571a9c4f..b06cc5e069c 100644 --- a/.github/workflows/issue-arborist.lock.yml +++ b/.github/workflows/issue-arborist.lock.yml @@ -1269,185 +1269,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/issue-monster.lock.yml b/.github/workflows/issue-monster.lock.yml index 5d96fe414e2..6b0f61c0171 100644 --- a/.github/workflows/issue-monster.lock.yml +++ b/.github/workflows/issue-monster.lock.yml @@ -1486,185 +1486,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/issue-triage-agent.lock.yml b/.github/workflows/issue-triage-agent.lock.yml index 9e1af2c683c..a9d30bf236d 100644 --- a/.github/workflows/issue-triage-agent.lock.yml +++ b/.github/workflows/issue-triage-agent.lock.yml @@ -1092,185 +1092,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/jsweep.lock.yml b/.github/workflows/jsweep.lock.yml index c4bd8c3b290..713a7fc65ee 100644 --- a/.github/workflows/jsweep.lock.yml +++ b/.github/workflows/jsweep.lock.yml @@ -1168,185 +1168,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/layout-spec-maintainer.lock.yml b/.github/workflows/layout-spec-maintainer.lock.yml index 894d4b2ff57..cce29b3b2b4 100644 --- a/.github/workflows/layout-spec-maintainer.lock.yml +++ b/.github/workflows/layout-spec-maintainer.lock.yml @@ -1156,185 +1156,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/lint-monster.lock.yml b/.github/workflows/lint-monster.lock.yml index 03265a7fadd..0c95b21ad2c 100644 --- a/.github/workflows/lint-monster.lock.yml +++ b/.github/workflows/lint-monster.lock.yml @@ -1201,185 +1201,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/linter-miner.lock.yml b/.github/workflows/linter-miner.lock.yml index f49b5b0e3e0..dd1ad367b4a 100644 --- a/.github/workflows/linter-miner.lock.yml +++ b/.github/workflows/linter-miner.lock.yml @@ -1198,185 +1198,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/lockfile-stats.lock.yml b/.github/workflows/lockfile-stats.lock.yml index 570d9a17896..d8f172a34aa 100644 --- a/.github/workflows/lockfile-stats.lock.yml +++ b/.github/workflows/lockfile-stats.lock.yml @@ -1207,185 +1207,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/mattpocock-skills-reviewer.lock.yml b/.github/workflows/mattpocock-skills-reviewer.lock.yml index 2ce6cc911c1..8b2283a37a3 100644 --- a/.github/workflows/mattpocock-skills-reviewer.lock.yml +++ b/.github/workflows/mattpocock-skills-reviewer.lock.yml @@ -1234,185 +1234,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/mcp-inspector.lock.yml b/.github/workflows/mcp-inspector.lock.yml index c84d7af875f..efed4553ec2 100644 --- a/.github/workflows/mcp-inspector.lock.yml +++ b/.github/workflows/mcp-inspector.lock.yml @@ -1677,185 +1677,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/mergefest.lock.yml b/.github/workflows/mergefest.lock.yml index 0a0f4a27760..75c63d10a20 100644 --- a/.github/workflows/mergefest.lock.yml +++ b/.github/workflows/mergefest.lock.yml @@ -1195,185 +1195,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/metrics-collector.lock.yml b/.github/workflows/metrics-collector.lock.yml index cfd0b3484d7..cce92b2ac63 100644 --- a/.github/workflows/metrics-collector.lock.yml +++ b/.github/workflows/metrics-collector.lock.yml @@ -1237,185 +1237,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/necromancer.lock.yml b/.github/workflows/necromancer.lock.yml index 0b81d927a76..9305c2ec595 100644 --- a/.github/workflows/necromancer.lock.yml +++ b/.github/workflows/necromancer.lock.yml @@ -1213,185 +1213,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/notion-issue-summary.lock.yml b/.github/workflows/notion-issue-summary.lock.yml index 7bf317ac1e5..a7a8d7ff1ba 100644 --- a/.github/workflows/notion-issue-summary.lock.yml +++ b/.github/workflows/notion-issue-summary.lock.yml @@ -1109,185 +1109,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/objective-impact-report.lock.yml b/.github/workflows/objective-impact-report.lock.yml index 585ae1267bb..089211617fb 100644 --- a/.github/workflows/objective-impact-report.lock.yml +++ b/.github/workflows/objective-impact-report.lock.yml @@ -1114,185 +1114,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/org-health-report.lock.yml b/.github/workflows/org-health-report.lock.yml index f47ce25bcfd..a5f82010a97 100644 --- a/.github/workflows/org-health-report.lock.yml +++ b/.github/workflows/org-health-report.lock.yml @@ -1210,185 +1210,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/outcome-collector.lock.yml b/.github/workflows/outcome-collector.lock.yml index c5100e1117f..ccdff6d64f2 100644 --- a/.github/workflows/outcome-collector.lock.yml +++ b/.github/workflows/outcome-collector.lock.yml @@ -1157,185 +1157,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/pdf-summary.lock.yml b/.github/workflows/pdf-summary.lock.yml index b547b667226..0d6410d53a0 100644 --- a/.github/workflows/pdf-summary.lock.yml +++ b/.github/workflows/pdf-summary.lock.yml @@ -1269,185 +1269,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/plan.lock.yml b/.github/workflows/plan.lock.yml index af03edd15d9..ece84082a54 100644 --- a/.github/workflows/plan.lock.yml +++ b/.github/workflows/plan.lock.yml @@ -1198,185 +1198,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/poem-bot.lock.yml b/.github/workflows/poem-bot.lock.yml index f0a2a71acf4..ea34fdccc4c 100644 --- a/.github/workflows/poem-bot.lock.yml +++ b/.github/workflows/poem-bot.lock.yml @@ -1480,185 +1480,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/portfolio-analyst.lock.yml b/.github/workflows/portfolio-analyst.lock.yml index 76f5854b758..1cda3b22eb1 100644 --- a/.github/workflows/portfolio-analyst.lock.yml +++ b/.github/workflows/portfolio-analyst.lock.yml @@ -1336,185 +1336,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/pr-code-quality-reviewer.lock.yml b/.github/workflows/pr-code-quality-reviewer.lock.yml index 12e5d5a63a6..71a0eb9bd63 100644 --- a/.github/workflows/pr-code-quality-reviewer.lock.yml +++ b/.github/workflows/pr-code-quality-reviewer.lock.yml @@ -1194,185 +1194,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/pr-description-caveman.lock.yml b/.github/workflows/pr-description-caveman.lock.yml index 4811ba3de56..7dc1cfb33a7 100644 --- a/.github/workflows/pr-description-caveman.lock.yml +++ b/.github/workflows/pr-description-caveman.lock.yml @@ -1116,185 +1116,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/pr-nitpick-reviewer.lock.yml b/.github/workflows/pr-nitpick-reviewer.lock.yml index 168e770142d..e7f0636584a 100644 --- a/.github/workflows/pr-nitpick-reviewer.lock.yml +++ b/.github/workflows/pr-nitpick-reviewer.lock.yml @@ -1238,185 +1238,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/pr-sous-chef.lock.yml b/.github/workflows/pr-sous-chef.lock.yml index bedac56ecd5..341708c126e 100644 --- a/.github/workflows/pr-sous-chef.lock.yml +++ b/.github/workflows/pr-sous-chef.lock.yml @@ -1236,185 +1236,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/pr-triage-agent.lock.yml b/.github/workflows/pr-triage-agent.lock.yml index e250673abae..63010023e38 100644 --- a/.github/workflows/pr-triage-agent.lock.yml +++ b/.github/workflows/pr-triage-agent.lock.yml @@ -1262,185 +1262,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/prompt-clustering-analysis.lock.yml b/.github/workflows/prompt-clustering-analysis.lock.yml index 586b886049f..a9d7fe4de18 100644 --- a/.github/workflows/prompt-clustering-analysis.lock.yml +++ b/.github/workflows/prompt-clustering-analysis.lock.yml @@ -1378,185 +1378,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/python-data-charts.lock.yml b/.github/workflows/python-data-charts.lock.yml index 68f1a1a56ac..a69a80577d7 100644 --- a/.github/workflows/python-data-charts.lock.yml +++ b/.github/workflows/python-data-charts.lock.yml @@ -1293,185 +1293,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/q.lock.yml b/.github/workflows/q.lock.yml index a298d7d14e4..16b610e28dd 100644 --- a/.github/workflows/q.lock.yml +++ b/.github/workflows/q.lock.yml @@ -1340,185 +1340,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/refactoring-cadence.lock.yml b/.github/workflows/refactoring-cadence.lock.yml index cc8e67c01cc..5394e84a3c7 100644 --- a/.github/workflows/refactoring-cadence.lock.yml +++ b/.github/workflows/refactoring-cadence.lock.yml @@ -1150,185 +1150,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/refiner.lock.yml b/.github/workflows/refiner.lock.yml index 151d0327f6d..62c87e3d9e0 100644 --- a/.github/workflows/refiner.lock.yml +++ b/.github/workflows/refiner.lock.yml @@ -1238,185 +1238,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/release.lock.yml b/.github/workflows/release.lock.yml index 6886c10a918..18aafb6e9ed 100644 --- a/.github/workflows/release.lock.yml +++ b/.github/workflows/release.lock.yml @@ -1152,185 +1152,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/repo-audit-analyzer.lock.yml b/.github/workflows/repo-audit-analyzer.lock.yml index 841559fa8fb..52f3dd249fd 100644 --- a/.github/workflows/repo-audit-analyzer.lock.yml +++ b/.github/workflows/repo-audit-analyzer.lock.yml @@ -1148,185 +1148,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/repo-tree-map.lock.yml b/.github/workflows/repo-tree-map.lock.yml index 34c0f8389f7..22851252455 100644 --- a/.github/workflows/repo-tree-map.lock.yml +++ b/.github/workflows/repo-tree-map.lock.yml @@ -1097,185 +1097,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/repository-quality-improver.lock.yml b/.github/workflows/repository-quality-improver.lock.yml index 106fbe4e308..26e643966e2 100644 --- a/.github/workflows/repository-quality-improver.lock.yml +++ b/.github/workflows/repository-quality-improver.lock.yml @@ -1151,185 +1151,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/research.lock.yml b/.github/workflows/research.lock.yml index d7cfca71dfa..e53d9422860 100644 --- a/.github/workflows/research.lock.yml +++ b/.github/workflows/research.lock.yml @@ -1127,185 +1127,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/ruflo-backed-task.lock.yml b/.github/workflows/ruflo-backed-task.lock.yml index 39a226a9f3d..6070f45177d 100644 --- a/.github/workflows/ruflo-backed-task.lock.yml +++ b/.github/workflows/ruflo-backed-task.lock.yml @@ -1306,185 +1306,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/safe-output-health.lock.yml b/.github/workflows/safe-output-health.lock.yml index e2f39a3bdc5..236188d4e2b 100644 --- a/.github/workflows/safe-output-health.lock.yml +++ b/.github/workflows/safe-output-health.lock.yml @@ -1323,185 +1323,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/schema-consistency-checker.lock.yml b/.github/workflows/schema-consistency-checker.lock.yml index ba78eafa036..f092f8699e5 100644 --- a/.github/workflows/schema-consistency-checker.lock.yml +++ b/.github/workflows/schema-consistency-checker.lock.yml @@ -1124,185 +1124,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/schema-feature-coverage.lock.yml b/.github/workflows/schema-feature-coverage.lock.yml index 5501f464df2..bc544783dd4 100644 --- a/.github/workflows/schema-feature-coverage.lock.yml +++ b/.github/workflows/schema-feature-coverage.lock.yml @@ -1168,185 +1168,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/scout.lock.yml b/.github/workflows/scout.lock.yml index f4dab02b99b..7ffb038838a 100644 --- a/.github/workflows/scout.lock.yml +++ b/.github/workflows/scout.lock.yml @@ -1405,185 +1405,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/security-compliance.lock.yml b/.github/workflows/security-compliance.lock.yml index d84fb686829..ff56f2ee61c 100644 --- a/.github/workflows/security-compliance.lock.yml +++ b/.github/workflows/security-compliance.lock.yml @@ -1156,185 +1156,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/security-review.lock.yml b/.github/workflows/security-review.lock.yml index adcacbffa60..66b667f044c 100644 --- a/.github/workflows/security-review.lock.yml +++ b/.github/workflows/security-review.lock.yml @@ -1282,185 +1282,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/semantic-function-refactor.lock.yml b/.github/workflows/semantic-function-refactor.lock.yml index 93c9a38f9e1..c5bda70b35e 100644 --- a/.github/workflows/semantic-function-refactor.lock.yml +++ b/.github/workflows/semantic-function-refactor.lock.yml @@ -1242,185 +1242,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/sergo.lock.yml b/.github/workflows/sergo.lock.yml index a967a821f06..d670fc734b8 100644 --- a/.github/workflows/sergo.lock.yml +++ b/.github/workflows/sergo.lock.yml @@ -1292,185 +1292,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/skillet.lock.yml b/.github/workflows/skillet.lock.yml index 16767f60b97..552c2d7e8bc 100644 --- a/.github/workflows/skillet.lock.yml +++ b/.github/workflows/skillet.lock.yml @@ -1208,185 +1208,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/slide-deck-maintainer.lock.yml b/.github/workflows/slide-deck-maintainer.lock.yml index bfdd1543892..a3696709435 100644 --- a/.github/workflows/slide-deck-maintainer.lock.yml +++ b/.github/workflows/slide-deck-maintainer.lock.yml @@ -1248,185 +1248,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-agent-all-merged.lock.yml b/.github/workflows/smoke-agent-all-merged.lock.yml index dfa90651bb0..3ed2ea00dbb 100644 --- a/.github/workflows/smoke-agent-all-merged.lock.yml +++ b/.github/workflows/smoke-agent-all-merged.lock.yml @@ -1208,185 +1208,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-agent-all-none.lock.yml b/.github/workflows/smoke-agent-all-none.lock.yml index bf36328f12a..37d5fed179b 100644 --- a/.github/workflows/smoke-agent-all-none.lock.yml +++ b/.github/workflows/smoke-agent-all-none.lock.yml @@ -1208,185 +1208,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-agent-public-approved.lock.yml b/.github/workflows/smoke-agent-public-approved.lock.yml index 8e7f523bf1c..8bc8731d833 100644 --- a/.github/workflows/smoke-agent-public-approved.lock.yml +++ b/.github/workflows/smoke-agent-public-approved.lock.yml @@ -1239,185 +1239,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-agent-public-none.lock.yml b/.github/workflows/smoke-agent-public-none.lock.yml index ea7af2e5a43..31d1371c24a 100644 --- a/.github/workflows/smoke-agent-public-none.lock.yml +++ b/.github/workflows/smoke-agent-public-none.lock.yml @@ -1208,185 +1208,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-agent-scoped-approved.lock.yml b/.github/workflows/smoke-agent-scoped-approved.lock.yml index d83947fb82e..d9cdfabb425 100644 --- a/.github/workflows/smoke-agent-scoped-approved.lock.yml +++ b/.github/workflows/smoke-agent-scoped-approved.lock.yml @@ -1215,185 +1215,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-antigravity.lock.yml b/.github/workflows/smoke-antigravity.lock.yml index d44ab7e8c79..75833d9f7bd 100644 --- a/.github/workflows/smoke-antigravity.lock.yml +++ b/.github/workflows/smoke-antigravity.lock.yml @@ -1276,185 +1276,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-call-workflow.lock.yml b/.github/workflows/smoke-call-workflow.lock.yml index 9bcfd88cc40..7f7048c603f 100644 --- a/.github/workflows/smoke-call-workflow.lock.yml +++ b/.github/workflows/smoke-call-workflow.lock.yml @@ -1210,185 +1210,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-ci.lock.yml b/.github/workflows/smoke-ci.lock.yml index c19aa4b9845..6ce33251253 100644 --- a/.github/workflows/smoke-ci.lock.yml +++ b/.github/workflows/smoke-ci.lock.yml @@ -1399,185 +1399,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-claude.lock.yml b/.github/workflows/smoke-claude.lock.yml index 59813476617..069c415341f 100644 --- a/.github/workflows/smoke-claude.lock.yml +++ b/.github/workflows/smoke-claude.lock.yml @@ -2043,185 +2043,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-codex.lock.yml b/.github/workflows/smoke-codex.lock.yml index 3f5639742f8..3b74ed4413c 100644 --- a/.github/workflows/smoke-codex.lock.yml +++ b/.github/workflows/smoke-codex.lock.yml @@ -1571,185 +1571,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-copilot-aoai-apikey.lock.yml b/.github/workflows/smoke-copilot-aoai-apikey.lock.yml index 059de7c4658..0624fe7e41b 100644 --- a/.github/workflows/smoke-copilot-aoai-apikey.lock.yml +++ b/.github/workflows/smoke-copilot-aoai-apikey.lock.yml @@ -2211,185 +2211,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-copilot-aoai-entra.lock.yml b/.github/workflows/smoke-copilot-aoai-entra.lock.yml index 0bd4fdeb442..c9693bcfa6e 100644 --- a/.github/workflows/smoke-copilot-aoai-entra.lock.yml +++ b/.github/workflows/smoke-copilot-aoai-entra.lock.yml @@ -2215,185 +2215,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-copilot-arm.lock.yml b/.github/workflows/smoke-copilot-arm.lock.yml index bf9fafcc608..a0835c474e8 100644 --- a/.github/workflows/smoke-copilot-arm.lock.yml +++ b/.github/workflows/smoke-copilot-arm.lock.yml @@ -2069,185 +2069,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-copilot-sdk.lock.yml b/.github/workflows/smoke-copilot-sdk.lock.yml index cfeebcaf51b..564f5e7a91e 100644 --- a/.github/workflows/smoke-copilot-sdk.lock.yml +++ b/.github/workflows/smoke-copilot-sdk.lock.yml @@ -1145,185 +1145,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-copilot.lock.yml b/.github/workflows/smoke-copilot.lock.yml index ab64744cbdb..a78f9591ca1 100644 --- a/.github/workflows/smoke-copilot.lock.yml +++ b/.github/workflows/smoke-copilot.lock.yml @@ -2213,185 +2213,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-create-cross-repo-pr.lock.yml b/.github/workflows/smoke-create-cross-repo-pr.lock.yml index 32fa92ff091..7020aefbbe2 100644 --- a/.github/workflows/smoke-create-cross-repo-pr.lock.yml +++ b/.github/workflows/smoke-create-cross-repo-pr.lock.yml @@ -1275,185 +1275,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-crush.lock.yml b/.github/workflows/smoke-crush.lock.yml index 3c71ed77c84..5933c41aca3 100644 --- a/.github/workflows/smoke-crush.lock.yml +++ b/.github/workflows/smoke-crush.lock.yml @@ -1174,185 +1174,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-gemini.lock.yml b/.github/workflows/smoke-gemini.lock.yml index a8ab6adf95e..abf75c265c0 100644 --- a/.github/workflows/smoke-gemini.lock.yml +++ b/.github/workflows/smoke-gemini.lock.yml @@ -1279,185 +1279,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-multi-pr.lock.yml b/.github/workflows/smoke-multi-pr.lock.yml index 01c900a6cf5..457a5f98af4 100644 --- a/.github/workflows/smoke-multi-pr.lock.yml +++ b/.github/workflows/smoke-multi-pr.lock.yml @@ -1220,185 +1220,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-opencode.lock.yml b/.github/workflows/smoke-opencode.lock.yml index d62bd4513c8..4d1c089d758 100644 --- a/.github/workflows/smoke-opencode.lock.yml +++ b/.github/workflows/smoke-opencode.lock.yml @@ -1179,185 +1179,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-otel-backends.lock.yml b/.github/workflows/smoke-otel-backends.lock.yml index 114c8b27ede..ce2e45965f2 100644 --- a/.github/workflows/smoke-otel-backends.lock.yml +++ b/.github/workflows/smoke-otel-backends.lock.yml @@ -1318,185 +1318,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-pi.lock.yml b/.github/workflows/smoke-pi.lock.yml index e58b535ec34..86daad57850 100644 --- a/.github/workflows/smoke-pi.lock.yml +++ b/.github/workflows/smoke-pi.lock.yml @@ -1232,185 +1232,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-project.lock.yml b/.github/workflows/smoke-project.lock.yml index dd55302223a..bcdb8ee06b3 100644 --- a/.github/workflows/smoke-project.lock.yml +++ b/.github/workflows/smoke-project.lock.yml @@ -1402,185 +1402,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-service-ports.lock.yml b/.github/workflows/smoke-service-ports.lock.yml index 18278b4151c..2335bebb97c 100644 --- a/.github/workflows/smoke-service-ports.lock.yml +++ b/.github/workflows/smoke-service-ports.lock.yml @@ -1146,185 +1146,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-temporary-id.lock.yml b/.github/workflows/smoke-temporary-id.lock.yml index ea71d210227..be6ee65e1fd 100644 --- a/.github/workflows/smoke-temporary-id.lock.yml +++ b/.github/workflows/smoke-temporary-id.lock.yml @@ -1247,185 +1247,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-test-tools.lock.yml b/.github/workflows/smoke-test-tools.lock.yml index e5a6ea3cfee..97721632ccb 100644 --- a/.github/workflows/smoke-test-tools.lock.yml +++ b/.github/workflows/smoke-test-tools.lock.yml @@ -1178,185 +1178,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-update-cross-repo-pr.lock.yml b/.github/workflows/smoke-update-cross-repo-pr.lock.yml index 2c15c3b8efa..6654917dea0 100644 --- a/.github/workflows/smoke-update-cross-repo-pr.lock.yml +++ b/.github/workflows/smoke-update-cross-repo-pr.lock.yml @@ -1306,185 +1306,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-workflow-call-with-inputs.lock.yml b/.github/workflows/smoke-workflow-call-with-inputs.lock.yml index 9def7dd8b7c..c084c32d5dd 100644 --- a/.github/workflows/smoke-workflow-call-with-inputs.lock.yml +++ b/.github/workflows/smoke-workflow-call-with-inputs.lock.yml @@ -1203,185 +1203,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/smoke-workflow-call.lock.yml b/.github/workflows/smoke-workflow-call.lock.yml index a036bb552ea..7196486434c 100644 --- a/.github/workflows/smoke-workflow-call.lock.yml +++ b/.github/workflows/smoke-workflow-call.lock.yml @@ -1192,185 +1192,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/spec-enforcer.lock.yml b/.github/workflows/spec-enforcer.lock.yml index 7ee758698cc..72059f73f0b 100644 --- a/.github/workflows/spec-enforcer.lock.yml +++ b/.github/workflows/spec-enforcer.lock.yml @@ -1138,185 +1138,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/spec-extractor.lock.yml b/.github/workflows/spec-extractor.lock.yml index 40cda971a58..acbb89a8847 100644 --- a/.github/workflows/spec-extractor.lock.yml +++ b/.github/workflows/spec-extractor.lock.yml @@ -1231,185 +1231,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/spec-librarian.lock.yml b/.github/workflows/spec-librarian.lock.yml index 5da845e8235..b9d7f4ab419 100644 --- a/.github/workflows/spec-librarian.lock.yml +++ b/.github/workflows/spec-librarian.lock.yml @@ -1192,185 +1192,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/stale-pr-cleanup.lock.yml b/.github/workflows/stale-pr-cleanup.lock.yml index fd251f92776..fb4ce9f2e0e 100644 --- a/.github/workflows/stale-pr-cleanup.lock.yml +++ b/.github/workflows/stale-pr-cleanup.lock.yml @@ -1139,185 +1139,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/stale-repo-identifier.lock.yml b/.github/workflows/stale-repo-identifier.lock.yml index 02a7bfee82f..af944b3360d 100644 --- a/.github/workflows/stale-repo-identifier.lock.yml +++ b/.github/workflows/stale-repo-identifier.lock.yml @@ -1335,185 +1335,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/static-analysis-report.lock.yml b/.github/workflows/static-analysis-report.lock.yml index 9c55c36b3c2..cd0b0c09f59 100644 --- a/.github/workflows/static-analysis-report.lock.yml +++ b/.github/workflows/static-analysis-report.lock.yml @@ -1348,185 +1348,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/step-name-alignment.lock.yml b/.github/workflows/step-name-alignment.lock.yml index b2eefd398de..2f7afde3b90 100644 --- a/.github/workflows/step-name-alignment.lock.yml +++ b/.github/workflows/step-name-alignment.lock.yml @@ -1234,185 +1234,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/sub-issue-closer.lock.yml b/.github/workflows/sub-issue-closer.lock.yml index 1ee8e2e4999..92045031756 100644 --- a/.github/workflows/sub-issue-closer.lock.yml +++ b/.github/workflows/sub-issue-closer.lock.yml @@ -1140,185 +1140,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/super-linter.lock.yml b/.github/workflows/super-linter.lock.yml index 5bb97fc4ecc..a9fb59c33e0 100644 --- a/.github/workflows/super-linter.lock.yml +++ b/.github/workflows/super-linter.lock.yml @@ -1168,185 +1168,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/technical-doc-writer.lock.yml b/.github/workflows/technical-doc-writer.lock.yml index c8f91974550..992c3bacc22 100644 --- a/.github/workflows/technical-doc-writer.lock.yml +++ b/.github/workflows/technical-doc-writer.lock.yml @@ -1258,185 +1258,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/terminal-stylist.lock.yml b/.github/workflows/terminal-stylist.lock.yml index b03f34a8dfa..d2c81655d78 100644 --- a/.github/workflows/terminal-stylist.lock.yml +++ b/.github/workflows/terminal-stylist.lock.yml @@ -1130,185 +1130,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/test-create-pr-error-handling.lock.yml b/.github/workflows/test-create-pr-error-handling.lock.yml index e735b3598d2..9c0ded84155 100644 --- a/.github/workflows/test-create-pr-error-handling.lock.yml +++ b/.github/workflows/test-create-pr-error-handling.lock.yml @@ -1215,185 +1215,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/test-dispatcher.lock.yml b/.github/workflows/test-dispatcher.lock.yml index 9e7231c9339..e8ba08566d1 100644 --- a/.github/workflows/test-dispatcher.lock.yml +++ b/.github/workflows/test-dispatcher.lock.yml @@ -1095,185 +1095,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/test-project-url-default.lock.yml b/.github/workflows/test-project-url-default.lock.yml index eab10b1e6f9..9cb60bb54d5 100644 --- a/.github/workflows/test-project-url-default.lock.yml +++ b/.github/workflows/test-project-url-default.lock.yml @@ -1141,185 +1141,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/test-quality-sentinel.lock.yml b/.github/workflows/test-quality-sentinel.lock.yml index 42d46637de7..c31d6b03be7 100644 --- a/.github/workflows/test-quality-sentinel.lock.yml +++ b/.github/workflows/test-quality-sentinel.lock.yml @@ -1208,185 +1208,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/test-workflow.lock.yml b/.github/workflows/test-workflow.lock.yml index cfa94c724d5..2b83bac8cce 100644 --- a/.github/workflows/test-workflow.lock.yml +++ b/.github/workflows/test-workflow.lock.yml @@ -1018,185 +1018,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/tidy.lock.yml b/.github/workflows/tidy.lock.yml index 29267e1b35b..cfdc79917c3 100644 --- a/.github/workflows/tidy.lock.yml +++ b/.github/workflows/tidy.lock.yml @@ -1237,185 +1237,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/typist.lock.yml b/.github/workflows/typist.lock.yml index e771ce385a0..71df2e98f1c 100644 --- a/.github/workflows/typist.lock.yml +++ b/.github/workflows/typist.lock.yml @@ -1255,185 +1255,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/ubuntu-image-analyzer.lock.yml b/.github/workflows/ubuntu-image-analyzer.lock.yml index 14906652379..34b0cb0e49c 100644 --- a/.github/workflows/ubuntu-image-analyzer.lock.yml +++ b/.github/workflows/ubuntu-image-analyzer.lock.yml @@ -1151,185 +1151,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/uk-ai-operational-resilience.lock.yml b/.github/workflows/uk-ai-operational-resilience.lock.yml index f295dbae6f6..55f6cd0849d 100644 --- a/.github/workflows/uk-ai-operational-resilience.lock.yml +++ b/.github/workflows/uk-ai-operational-resilience.lock.yml @@ -1131,185 +1131,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/unbloat-docs.lock.yml b/.github/workflows/unbloat-docs.lock.yml index a9162b13ffd..6794298cb98 100644 --- a/.github/workflows/unbloat-docs.lock.yml +++ b/.github/workflows/unbloat-docs.lock.yml @@ -1229,185 +1229,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/update-astro.lock.yml b/.github/workflows/update-astro.lock.yml index 8fecbe5d9e2..34e47700a90 100644 --- a/.github/workflows/update-astro.lock.yml +++ b/.github/workflows/update-astro.lock.yml @@ -1176,185 +1176,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/video-analyzer.lock.yml b/.github/workflows/video-analyzer.lock.yml index ad582a8e4bd..b491d464112 100644 --- a/.github/workflows/video-analyzer.lock.yml +++ b/.github/workflows/video-analyzer.lock.yml @@ -1118,185 +1118,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/visual-regression-checker.lock.yml b/.github/workflows/visual-regression-checker.lock.yml index 5776da729e2..e28d159247b 100644 --- a/.github/workflows/visual-regression-checker.lock.yml +++ b/.github/workflows/visual-regression-checker.lock.yml @@ -1188,185 +1188,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/weekly-blog-post-writer.lock.yml b/.github/workflows/weekly-blog-post-writer.lock.yml index acc256ac414..f6c079671b4 100644 --- a/.github/workflows/weekly-blog-post-writer.lock.yml +++ b/.github/workflows/weekly-blog-post-writer.lock.yml @@ -1316,185 +1316,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/weekly-editors-health-check.lock.yml b/.github/workflows/weekly-editors-health-check.lock.yml index 003d8405e8e..101a697c4e3 100644 --- a/.github/workflows/weekly-editors-health-check.lock.yml +++ b/.github/workflows/weekly-editors-health-check.lock.yml @@ -1185,185 +1185,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/weekly-issue-summary.lock.yml b/.github/workflows/weekly-issue-summary.lock.yml index 36f9217f50f..5b08b49eb60 100644 --- a/.github/workflows/weekly-issue-summary.lock.yml +++ b/.github/workflows/weekly-issue-summary.lock.yml @@ -1173,185 +1173,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/weekly-safe-outputs-spec-review.lock.yml b/.github/workflows/weekly-safe-outputs-spec-review.lock.yml index 97fa3f8b22b..847f532ee1e 100644 --- a/.github/workflows/weekly-safe-outputs-spec-review.lock.yml +++ b/.github/workflows/weekly-safe-outputs-spec-review.lock.yml @@ -1108,185 +1108,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/workflow-generator.lock.yml b/.github/workflows/workflow-generator.lock.yml index 7a72dcbedc8..7c3bc108c11 100644 --- a/.github/workflows/workflow-generator.lock.yml +++ b/.github/workflows/workflow-generator.lock.yml @@ -1181,185 +1181,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/workflow-health-manager.lock.yml b/.github/workflows/workflow-health-manager.lock.yml index fd479f7e791..52fb2e6fd23 100644 --- a/.github/workflows/workflow-health-manager.lock.yml +++ b/.github/workflows/workflow-health-manager.lock.yml @@ -1227,185 +1227,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/workflow-normalizer.lock.yml b/.github/workflows/workflow-normalizer.lock.yml index 545897b8f5c..aef86f958a1 100644 --- a/.github/workflows/workflow-normalizer.lock.yml +++ b/.github/workflows/workflow-normalizer.lock.yml @@ -1191,185 +1191,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/.github/workflows/workflow-skill-extractor.lock.yml b/.github/workflows/workflow-skill-extractor.lock.yml index 2c9235c539e..5bb2ed31fbd 100644 --- a/.github/workflows/workflow-skill-extractor.lock.yml +++ b/.github/workflows/workflow-skill-extractor.lock.yml @@ -1162,185 +1162,7 @@ jobs: [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl mkdir -p /tmp/gh-aw/usage/activity - python3 - <<'PY' - import glob - import json - import os - - # NOTE: this aggregation script intentionally stays inline in the generated - # workflow step so compiled workflows are self-contained and do not depend on - # extra repository files at runtime. - # usage-activity-summary/v1 structure: - # firewall: total/allowed/blocked request counters - # session: aggregate Copilot session event counters - # gateway: total/failed tool-call counters with per-server breakdown - summary = {'schema': 'usage-activity-summary/v1'} - SQUID_STATUS_INDEX = 6 - SQUID_DECISION_INDEX = 7 - - firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0} - def is_allowed_decision(decision: str) -> bool: - base = decision.split('/', 1)[0].strip().upper() - return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS') - - firewall_paths = [ - '/tmp/gh-aw/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log', - '/tmp/gh-aw/squid-logs-*/*.log', - '/tmp/gh-aw/threat-detection/squid-logs-*/*.log', - ] - for pattern in firewall_paths: - for log_path in glob.glob(pattern): - try: - with open(log_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - if len(parts) < 8: - continue - firewall['total_requests'] += 1 - # Squid access log columns (0-based): - # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method - # 6=status 7=decision 8=url 9=user-agent - # Keep indices named for easier maintenance if format changes. - status = parts[SQUID_STATUS_INDEX] - decision = parts[SQUID_DECISION_INDEX] - allowed = False - try: - code = int(status) - allowed = code in (200, 206, 304) - except ValueError: - allowed = False - if not allowed and is_allowed_decision(decision): - allowed = True - if allowed: - firewall['allowed_requests'] += 1 - else: - firewall['blocked_requests'] += 1 - except OSError: - continue - if firewall['total_requests'] > 0: - summary['firewall'] = firewall - - session = { - 'total_events': 0, - 'session_starts': 0, - 'session_shutdowns': 0, - 'turns': 0, - 'assistant_messages': 0, - 'reasoning_events': 0, - 'tool_execution_starts': 0, - 'tool_execution_completes': 0, - 'failed_tool_executions': 0, - } - session_paths = [ - '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl', - ] - for pattern in session_paths: - for events_path in glob.glob(pattern): - try: - with open(events_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event_type = str(entry.get('type', '')).strip().lower() - session['total_events'] += 1 - if event_type == 'session.start': - session['session_starts'] += 1 - elif event_type == 'session.shutdown': - session['session_shutdowns'] += 1 - elif event_type == 'user.message': - session['turns'] += 1 - elif event_type == 'assistant.message': - session['assistant_messages'] += 1 - # Copilot session logs use both reasoning and assistant.reasoning - # across CLI/runtime versions, so count both as reasoning events. - elif event_type in ('reasoning', 'assistant.reasoning'): - session['reasoning_events'] += 1 - elif event_type == 'tool.execution_start': - session['tool_execution_starts'] += 1 - elif event_type == 'tool.execution_complete': - session['tool_execution_completes'] += 1 - data = entry.get('data', {}) - success = True - if isinstance(data, dict): - success = bool(data.get('success', True)) - if not success: - session['failed_tool_executions'] += 1 - except OSError: - continue - if session['total_events'] > 0: - summary['session'] = session - - gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}} - gateway_paths = [] - for modern_path, legacy_path in [ - ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'), - ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'), - ]: - if os.path.exists(modern_path): - gateway_paths.append(modern_path) - elif os.path.exists(legacy_path): - gateway_paths.append(legacy_path) - for gateway_path in gateway_paths: - if not os.path.exists(gateway_path): - continue - try: - with open(gateway_path, encoding='utf-8', errors='ignore') as handle: - for raw in handle: - line = raw.strip() - if not line or not line.startswith('{'): - continue - try: - entry = json.loads(line) - except json.JSONDecodeError: - continue - event = str(entry.get('event', '')).strip().lower() - if event not in ('tool_call', 'rpc_call', 'request'): - continue - gateway['total_calls'] += 1 - status = str(entry.get('status', '')).strip().lower() - level = str(entry.get('level', '')).strip().lower() - error_text = str(entry.get('error', '')).strip() - failed = status == 'error' or error_text != '' or level == 'error' - if failed: - gateway['failed_calls'] += 1 - # gateway.jsonl has server_name for modern logs and server_id in - # some compatibility/transition paths; keep fallback ordering explicit. - server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown') - server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0}) - server_bucket['tool_call_count'] += 1 - if failed: - server_bucket['failed_calls'] += 1 - except OSError: - continue - if gateway['total_calls'] > 0: - summary['gateway'] = { - 'total_calls': gateway['total_calls'], - 'failed_calls': gateway['failed_calls'], - 'servers': [ - { - 'server_name': server_name, - 'tool_call_count': bucket['tool_call_count'], - 'failed_calls': bucket['failed_calls'], - } - for server_name, bucket in sorted(gateway['servers'].items()) - ], - } - - output_path = '/tmp/gh-aw/usage/activity/summary.json' - with open(output_path, 'w', encoding='utf-8') as handle: - json.dump(summary, handle, sort_keys=True) - print(output_path) - PY + node ${{ runner.temp }}/gh-aw/actions/generate_usage_activity_summary.cjs find /tmp/gh-aw/usage -type f -print | sort - name: Upload usage artifact if: always() diff --git a/actions/setup/js/generate_usage_activity_summary.cjs b/actions/setup/js/generate_usage_activity_summary.cjs new file mode 100644 index 00000000000..426405d1bcc --- /dev/null +++ b/actions/setup/js/generate_usage_activity_summary.cjs @@ -0,0 +1,298 @@ +#!/usr/bin/env node + +// This script aggregates usage activity data from various log sources and generates +// a compact summary.json file for the usage artifact. +// usage-activity-summary/v1 structure: +// firewall: total/allowed/blocked request counters +// session: aggregate Copilot session event counters +// gateway: total/failed tool-call counters with per-server breakdown + +const fs = require("fs"); +const path = require("path"); +const { globSync } = require("glob"); + +const SQUID_STATUS_INDEX = 6; +const SQUID_DECISION_INDEX = 7; + +/** + * Check if a Squid decision indicates an allowed request + */ +function isAllowedDecision(decision) { + const base = decision.split("/")[0].trim().toUpperCase(); + return ["TCP_TUNNEL", "TCP_HIT", "TCP_MISS"].includes(base); +} + +/** + * Parse firewall logs and aggregate request counts + */ +function parseFirewallLogs() { + const firewall = { total_requests: 0, allowed_requests: 0, blocked_requests: 0 }; + + const firewallPaths = ["/tmp/gh-aw/sandbox/firewall/logs/*.log", "/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log", "/tmp/gh-aw/squid-logs-*/*.log", "/tmp/gh-aw/threat-detection/squid-logs-*/*.log"]; + + for (const pattern of firewallPaths) { + const files = globSync(pattern, { nodir: true }); + for (const logPath of files) { + try { + const content = fs.readFileSync(logPath, "utf-8"); + const lines = content.split("\n"); + + for (const raw of lines) { + const line = raw.trim(); + if (!line || line.startsWith("#")) { + continue; + } + + const parts = line.split(/\s+/); + if (parts.length < 8) { + continue; + } + + firewall.total_requests += 1; + + // Squid access log columns (0-based): + // 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method + // 6=status 7=decision 8=url 9=user-agent + // Keep indices named for easier maintenance if format changes. + const status = parts[SQUID_STATUS_INDEX]; + const decision = parts[SQUID_DECISION_INDEX]; + + let allowed = false; + const code = parseInt(status, 10); + if (!isNaN(code) && [200, 206, 304].includes(code)) { + allowed = true; + } + + if (!allowed && isAllowedDecision(decision)) { + allowed = true; + } + + if (allowed) { + firewall.allowed_requests += 1; + } else { + firewall.blocked_requests += 1; + } + } + } catch (err) { + // Skip files that can't be read + continue; + } + } + } + + return firewall.total_requests > 0 ? firewall : null; +} + +/** + * Parse Copilot session event logs and aggregate counters + */ +function parseSessionLogs() { + const session = { + total_events: 0, + session_starts: 0, + session_shutdowns: 0, + turns: 0, + assistant_messages: 0, + reasoning_events: 0, + tool_execution_starts: 0, + tool_execution_completes: 0, + failed_tool_executions: 0, + }; + + const sessionPaths = ["/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl", "/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl"]; + + for (const pattern of sessionPaths) { + const files = globSync(pattern, { nodir: true }); + for (const eventsPath of files) { + try { + const content = fs.readFileSync(eventsPath, "utf-8"); + const lines = content.split("\n"); + + for (const raw of lines) { + const line = raw.trim(); + if (!line || !line.startsWith("{")) { + continue; + } + + let entry; + try { + entry = JSON.parse(line); + } catch { + continue; + } + + const eventType = String(entry.type || "") + .trim() + .toLowerCase(); + session.total_events += 1; + + if (eventType === "session.start") { + session.session_starts += 1; + } else if (eventType === "session.shutdown") { + session.session_shutdowns += 1; + } else if (eventType === "user.message") { + session.turns += 1; + } else if (eventType === "assistant.message") { + session.assistant_messages += 1; + } + // Copilot session logs use both reasoning and assistant.reasoning + // across CLI/runtime versions, so count both as reasoning events. + else if (eventType === "reasoning" || eventType === "assistant.reasoning") { + session.reasoning_events += 1; + } else if (eventType === "tool.execution_start") { + session.tool_execution_starts += 1; + } else if (eventType === "tool.execution_complete") { + session.tool_execution_completes += 1; + const data = entry.data || {}; + const success = typeof data === "object" ? data.success !== false : true; + if (!success) { + session.failed_tool_executions += 1; + } + } + } + } catch (err) { + // Skip files that can't be read + continue; + } + } + } + + return session.total_events > 0 ? session : null; +} + +/** + * Parse MCP gateway logs and aggregate tool call counts + */ +function parseGatewayLogs() { + const gateway = { total_calls: 0, failed_calls: 0, servers: {} }; + const gatewayPaths = []; + + const pathPairs = [ + ["/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl", "/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl"], + ["/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl", "/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl"], + ]; + + for (const [modernPath, legacyPath] of pathPairs) { + if (fs.existsSync(modernPath)) { + gatewayPaths.push(modernPath); + } else if (fs.existsSync(legacyPath)) { + gatewayPaths.push(legacyPath); + } + } + + for (const gatewayPath of gatewayPaths) { + if (!fs.existsSync(gatewayPath)) { + continue; + } + + try { + const content = fs.readFileSync(gatewayPath, "utf-8"); + const lines = content.split("\n"); + + for (const raw of lines) { + const line = raw.trim(); + if (!line || !line.startsWith("{")) { + continue; + } + + let entry; + try { + entry = JSON.parse(line); + } catch { + continue; + } + + const event = String(entry.event || "") + .trim() + .toLowerCase(); + if (!["tool_call", "rpc_call", "request"].includes(event)) { + continue; + } + + gateway.total_calls += 1; + + const status = String(entry.status || "") + .trim() + .toLowerCase(); + const level = String(entry.level || "") + .trim() + .toLowerCase(); + const errorText = String(entry.error || "").trim(); + const failed = status === "error" || errorText !== "" || level === "error"; + + if (failed) { + gateway.failed_calls += 1; + } + + // gateway.jsonl has server_name for modern logs and server_id in + // some compatibility/transition paths; keep fallback ordering explicit. + const serverName = String(entry.server_name || entry.server_id || "unknown"); + + if (!gateway.servers[serverName]) { + gateway.servers[serverName] = { tool_call_count: 0, failed_calls: 0 }; + } + + gateway.servers[serverName].tool_call_count += 1; + if (failed) { + gateway.servers[serverName].failed_calls += 1; + } + } + } catch (err) { + // Skip files that can't be read + continue; + } + } + + if (gateway.total_calls > 0) { + return { + total_calls: gateway.total_calls, + failed_calls: gateway.failed_calls, + servers: Object.entries(gateway.servers) + .sort((a, b) => a[0].localeCompare(b[0])) + .map(([serverName, bucket]) => ({ + server_name: serverName, + tool_call_count: bucket.tool_call_count, + failed_calls: bucket.failed_calls, + })), + }; + } + + return null; +} + +/** + * Main function to generate usage activity summary + */ +function main() { + const summary = { schema: "usage-activity-summary/v1" }; + + // Parse firewall logs + const firewall = parseFirewallLogs(); + if (firewall) { + summary.firewall = firewall; + } + + // Parse session logs + const session = parseSessionLogs(); + if (session) { + summary.session = session; + } + + // Parse gateway logs + const gateway = parseGatewayLogs(); + if (gateway) { + summary.gateway = gateway; + } + + // Write summary to file + const outputPath = "/tmp/gh-aw/usage/activity/summary.json"; + fs.writeFileSync(outputPath, JSON.stringify(summary, null, 2), "utf-8"); + console.log(outputPath); +} + +// Run main function +if (require.main === module) { + main(); +} + +module.exports = { parseFirewallLogs, parseSessionLogs, parseGatewayLogs }; diff --git a/pkg/workflow/notify_comment.go b/pkg/workflow/notify_comment.go index 27df841eaba..63298c63b1e 100644 --- a/pkg/workflow/notify_comment.go +++ b/pkg/workflow/notify_comment.go @@ -705,185 +705,7 @@ func buildUsageArtifactUploadSteps(prefix string, pinAction func(string) string) " [ -f /tmp/gh-aw/usage/agent/token_usage.jsonl ] || : > /tmp/gh-aw/usage/agent/token_usage.jsonl\n", " [ -f /tmp/gh-aw/usage/detection/token_usage.jsonl ] || : > /tmp/gh-aw/usage/detection/token_usage.jsonl\n", " mkdir -p /tmp/gh-aw/usage/activity\n", - " python3 - <<'PY'\n", - " import glob\n", - " import json\n", - " import os\n", - "\n", - " # NOTE: this aggregation script intentionally stays inline in the generated\n", - " # workflow step so compiled workflows are self-contained and do not depend on\n", - " # extra repository files at runtime.\n", - " # usage-activity-summary/v1 structure:\n", - " # firewall: total/allowed/blocked request counters\n", - " # session: aggregate Copilot session event counters\n", - " # gateway: total/failed tool-call counters with per-server breakdown\n", - " summary = {'schema': 'usage-activity-summary/v1'}\n", - " SQUID_STATUS_INDEX = 6\n", - " SQUID_DECISION_INDEX = 7\n", - "\n", - " firewall = {'total_requests': 0, 'allowed_requests': 0, 'blocked_requests': 0}\n", - " def is_allowed_decision(decision: str) -> bool:\n", - " base = decision.split('/', 1)[0].strip().upper()\n", - " return base in ('TCP_TUNNEL', 'TCP_HIT', 'TCP_MISS')\n", - "\n", - " firewall_paths = [\n", - " '/tmp/gh-aw/sandbox/firewall/logs/*.log',\n", - " '/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log',\n", - " '/tmp/gh-aw/squid-logs-*/*.log',\n", - " '/tmp/gh-aw/threat-detection/squid-logs-*/*.log',\n", - " ]\n", - " for pattern in firewall_paths:\n", - " for log_path in glob.glob(pattern):\n", - " try:\n", - " with open(log_path, encoding='utf-8', errors='ignore') as handle:\n", - " for raw in handle:\n", - " line = raw.strip()\n", - " if not line or line.startswith('#'):\n", - " continue\n", - " parts = line.split()\n", - " if len(parts) < 8:\n", - " continue\n", - " firewall['total_requests'] += 1\n", - " # Squid access log columns (0-based):\n", - " # 0=timestamp 1=client 2=domain 3=dest 4=proto 5=method\n", - " # 6=status 7=decision 8=url 9=user-agent\n", - " # Keep indices named for easier maintenance if format changes.\n", - " status = parts[SQUID_STATUS_INDEX]\n", - " decision = parts[SQUID_DECISION_INDEX]\n", - " allowed = False\n", - " try:\n", - " code = int(status)\n", - " allowed = code in (200, 206, 304)\n", - " except ValueError:\n", - " allowed = False\n", - " if not allowed and is_allowed_decision(decision):\n", - " allowed = True\n", - " if allowed:\n", - " firewall['allowed_requests'] += 1\n", - " else:\n", - " firewall['blocked_requests'] += 1\n", - " except OSError:\n", - " continue\n", - " if firewall['total_requests'] > 0:\n", - " summary['firewall'] = firewall\n", - "\n", - " session = {\n", - " 'total_events': 0,\n", - " 'session_starts': 0,\n", - " 'session_shutdowns': 0,\n", - " 'turns': 0,\n", - " 'assistant_messages': 0,\n", - " 'reasoning_events': 0,\n", - " 'tool_execution_starts': 0,\n", - " 'tool_execution_completes': 0,\n", - " 'failed_tool_executions': 0,\n", - " }\n", - " session_paths = [\n", - " '/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl',\n", - " '/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl',\n", - " ]\n", - " for pattern in session_paths:\n", - " for events_path in glob.glob(pattern):\n", - " try:\n", - " with open(events_path, encoding='utf-8', errors='ignore') as handle:\n", - " for raw in handle:\n", - " line = raw.strip()\n", - " if not line or not line.startswith('{'):\n", - " continue\n", - " try:\n", - " entry = json.loads(line)\n", - " except json.JSONDecodeError:\n", - " continue\n", - " event_type = str(entry.get('type', '')).strip().lower()\n", - " session['total_events'] += 1\n", - " if event_type == 'session.start':\n", - " session['session_starts'] += 1\n", - " elif event_type == 'session.shutdown':\n", - " session['session_shutdowns'] += 1\n", - " elif event_type == 'user.message':\n", - " session['turns'] += 1\n", - " elif event_type == 'assistant.message':\n", - " session['assistant_messages'] += 1\n", - " # Copilot session logs use both reasoning and assistant.reasoning\n", - " # across CLI/runtime versions, so count both as reasoning events.\n", - " elif event_type in ('reasoning', 'assistant.reasoning'):\n", - " session['reasoning_events'] += 1\n", - " elif event_type == 'tool.execution_start':\n", - " session['tool_execution_starts'] += 1\n", - " elif event_type == 'tool.execution_complete':\n", - " session['tool_execution_completes'] += 1\n", - " data = entry.get('data', {})\n", - " success = True\n", - " if isinstance(data, dict):\n", - " success = bool(data.get('success', True))\n", - " if not success:\n", - " session['failed_tool_executions'] += 1\n", - " except OSError:\n", - " continue\n", - " if session['total_events'] > 0:\n", - " summary['session'] = session\n", - "\n", - " gateway = {'total_calls': 0, 'failed_calls': 0, 'servers': {}}\n", - " gateway_paths = []\n", - " for modern_path, legacy_path in [\n", - " ('/tmp/gh-aw/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/sandbox/agent/logs/gateway.jsonl'),\n", - " ('/tmp/gh-aw/threat-detection/sandbox/agent/logs/mcp-logs/gateway.jsonl', '/tmp/gh-aw/threat-detection/sandbox/agent/logs/gateway.jsonl'),\n", - " ]:\n", - " if os.path.exists(modern_path):\n", - " gateway_paths.append(modern_path)\n", - " elif os.path.exists(legacy_path):\n", - " gateway_paths.append(legacy_path)\n", - " for gateway_path in gateway_paths:\n", - " if not os.path.exists(gateway_path):\n", - " continue\n", - " try:\n", - " with open(gateway_path, encoding='utf-8', errors='ignore') as handle:\n", - " for raw in handle:\n", - " line = raw.strip()\n", - " if not line or not line.startswith('{'):\n", - " continue\n", - " try:\n", - " entry = json.loads(line)\n", - " except json.JSONDecodeError:\n", - " continue\n", - " event = str(entry.get('event', '')).strip().lower()\n", - " if event not in ('tool_call', 'rpc_call', 'request'):\n", - " continue\n", - " gateway['total_calls'] += 1\n", - " status = str(entry.get('status', '')).strip().lower()\n", - " level = str(entry.get('level', '')).strip().lower()\n", - " error_text = str(entry.get('error', '')).strip()\n", - " failed = status == 'error' or error_text != '' or level == 'error'\n", - " if failed:\n", - " gateway['failed_calls'] += 1\n", - " # gateway.jsonl has server_name for modern logs and server_id in\n", - " # some compatibility/transition paths; keep fallback ordering explicit.\n", - " server_name = str(entry.get('server_name') or entry.get('server_id') or 'unknown')\n", - " server_bucket = gateway['servers'].setdefault(server_name, {'tool_call_count': 0, 'failed_calls': 0})\n", - " server_bucket['tool_call_count'] += 1\n", - " if failed:\n", - " server_bucket['failed_calls'] += 1\n", - " except OSError:\n", - " continue\n", - " if gateway['total_calls'] > 0:\n", - " summary['gateway'] = {\n", - " 'total_calls': gateway['total_calls'],\n", - " 'failed_calls': gateway['failed_calls'],\n", - " 'servers': [\n", - " {\n", - " 'server_name': server_name,\n", - " 'tool_call_count': bucket['tool_call_count'],\n", - " 'failed_calls': bucket['failed_calls'],\n", - " }\n", - " for server_name, bucket in sorted(gateway['servers'].items())\n", - " ],\n", - " }\n", - "\n", - " output_path = '/tmp/gh-aw/usage/activity/summary.json'\n", - " with open(output_path, 'w', encoding='utf-8') as handle:\n", - " json.dump(summary, handle, sort_keys=True)\n", - " print(output_path)\n", - " PY\n", + fmt.Sprintf(" node %s/generate_usage_activity_summary.cjs\n", SetupActionDestination), " find /tmp/gh-aw/usage -type f -print | sort\n", " - name: Upload usage artifact\n", " if: always()\n", diff --git a/pkg/workflow/notify_comment_test.go b/pkg/workflow/notify_comment_test.go index 7c99dcf9a59..c6c184848fa 100644 --- a/pkg/workflow/notify_comment_test.go +++ b/pkg/workflow/notify_comment_test.go @@ -1216,15 +1216,9 @@ func TestConclusionJobIncludesUsageArtifactSteps(t *testing.T) { if !strings.Contains(allSteps, ": > /tmp/gh-aw/usage/detection/token_usage.jsonl") { t.Errorf("Expected usage artifact collection to ensure detection token usage file exists.\nGenerated steps:\n%s", allSteps) } - if !strings.Contains(allSteps, "python3 - <<'PY'") { + if !strings.Contains(allSteps, "generate_usage_activity_summary.cjs") { t.Errorf("Expected usage artifact collection to generate activity summary aggregates.\nGenerated steps:\n%s", allSteps) } - if !strings.Contains(allSteps, "usage-activity-summary/v1") { - t.Errorf("Expected activity summary generator to emit the usage activity schema marker.\nGenerated steps:\n%s", allSteps) - } - if !strings.Contains(allSteps, "def is_allowed_decision(decision: str) -> bool:") { - t.Errorf("Expected activity summary generator to normalize Squid decision markers before counting allowed requests.\nGenerated steps:\n%s", allSteps) - } if !strings.Contains(allSteps, "/tmp/gh-aw/usage/activity/summary.json") { t.Errorf("Expected usage artifact to include activity summary path.\nGenerated steps:\n%s", allSteps) } From e0bf0a7ea4be7a4a2cfe0a11a2c28b1a437e152a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 21 Jun 2026 03:36:10 +0000 Subject: [PATCH 15/17] Use Node 24 builtin glob instead of external package Replaced `require('glob')` with `require('node:fs')` to use Node.js 24's builtin globSync function. This removes the dependency on the external glob npm package and uses the native Node.js API instead. Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/generate_usage_activity_summary.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/setup/js/generate_usage_activity_summary.cjs b/actions/setup/js/generate_usage_activity_summary.cjs index 426405d1bcc..2e3cceda82b 100644 --- a/actions/setup/js/generate_usage_activity_summary.cjs +++ b/actions/setup/js/generate_usage_activity_summary.cjs @@ -8,8 +8,8 @@ // gateway: total/failed tool-call counters with per-server breakdown const fs = require("fs"); +const { globSync } = require("node:fs"); const path = require("path"); -const { globSync } = require("glob"); const SQUID_STATUS_INDEX = 6; const SQUID_DECISION_INDEX = 7; From 45ffadebdbaafbe016eab8087f86aa4d360c26f2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 21 Jun 2026 03:56:06 +0000 Subject: [PATCH 16/17] Add changeset --- .changeset/minor-precompute-usage-activity-reporting.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/minor-precompute-usage-activity-reporting.md diff --git a/.changeset/minor-precompute-usage-activity-reporting.md b/.changeset/minor-precompute-usage-activity-reporting.md new file mode 100644 index 00000000000..bedf37c98d6 --- /dev/null +++ b/.changeset/minor-precompute-usage-activity-reporting.md @@ -0,0 +1,5 @@ +--- +"gh-aw": minor +--- + +Precompute usage-artifact activity aggregates and include GitHub API rate limit data so `gh aw logs` can produce richer usage-only activity reporting. From 4abc4dddb8ea2f670f8970c351b86f8fb69c8132 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 21 Jun 2026 04:52:15 +0000 Subject: [PATCH 17/17] Fix typecheck: remove unsupported nodir option from globSync Node.js 24's builtin globSync from node:fs doesn't support the nodir option. Removed it since the patterns (*.log, *.jsonl) already match only files, not directories. This fixes the TypeScript typecheck errors. Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/generate_usage_activity_summary.cjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/actions/setup/js/generate_usage_activity_summary.cjs b/actions/setup/js/generate_usage_activity_summary.cjs index 2e3cceda82b..8521eac0599 100644 --- a/actions/setup/js/generate_usage_activity_summary.cjs +++ b/actions/setup/js/generate_usage_activity_summary.cjs @@ -31,7 +31,7 @@ function parseFirewallLogs() { const firewallPaths = ["/tmp/gh-aw/sandbox/firewall/logs/*.log", "/tmp/gh-aw/threat-detection/sandbox/firewall/logs/*.log", "/tmp/gh-aw/squid-logs-*/*.log", "/tmp/gh-aw/threat-detection/squid-logs-*/*.log"]; for (const pattern of firewallPaths) { - const files = globSync(pattern, { nodir: true }); + const files = globSync(pattern); for (const logPath of files) { try { const content = fs.readFileSync(logPath, "utf-8"); @@ -102,7 +102,7 @@ function parseSessionLogs() { const sessionPaths = ["/tmp/gh-aw/sandbox/agent/logs/copilot-session-state/*/events.jsonl", "/tmp/gh-aw/threat-detection/sandbox/agent/logs/copilot-session-state/*/events.jsonl"]; for (const pattern of sessionPaths) { - const files = globSync(pattern, { nodir: true }); + const files = globSync(pattern); for (const eventsPath of files) { try { const content = fs.readFileSync(eventsPath, "utf-8");