diff --git a/tool/tctl/common/top/model.go b/tool/tctl/common/top/model.go index be7debcd6105e..dd4668a197d64 100644 --- a/tool/tctl/common/top/model.go +++ b/tool/tctl/common/top/model.go @@ -103,6 +103,8 @@ func (m *topModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.selected = 2 case "4": m.selected = 3 + case "5": + m.selected = 4 case "right": m.selected = min(m.selected+1, len(tabs)-1) case "left": @@ -218,6 +220,8 @@ func (m *topModel) contentView() string { return renderCache(m.report, m.height, m.width) case 3: return renderWatcher(m.report, m.height, m.width) + case 4: + return renderAudit(m.report, m.height, m.width) default: return "" } @@ -412,6 +416,7 @@ func renderWatcher(report *Report, height, width int) string { eventData, asciigraph.Height(graphHeight), asciigraph.Width(graphWidth-15), + asciigraph.UpperBound(1), ) eventCountContent := boxedView("Events/Sec", countPlot, graphWidth) @@ -423,6 +428,7 @@ func renderWatcher(report *Report, height, width int) string { sizeData, asciigraph.Height(graphHeight), asciigraph.Width(graphWidth-15), + asciigraph.UpperBound(1), ) eventSizeContent := boxedView("Bytes/Sec", sizePlot, graphWidth) @@ -449,6 +455,54 @@ func renderWatcher(report *Report, height, width int) string { ) } +// renderAudit generates the view for the audit stats tab. +func renderAudit(report *Report, height, width int) string { + graphHeight := height / 3 + graphWidth := width + + eventsLegend := lipgloss.JoinHorizontal( + lipgloss.Left, + "- Emitted", + failedEventStyle.Render(" - Failed"), + trimmedEventStyle.Render(" - Trimmed"), + ) + + eventsPlot := asciigraph.PlotMany( + [][]float64{ + report.Audit.EmittedEventsBuffer.Data(graphWidth - 15), + report.Audit.FailedEventsBuffer.Data(graphWidth - 15), + report.Audit.TrimmedEventsBuffer.Data(graphWidth - 15), + }, + asciigraph.Height(graphHeight), + asciigraph.Width(graphWidth-15), + asciigraph.UpperBound(1), + asciigraph.SeriesColors(asciigraph.Default, asciigraph.Red, asciigraph.Goldenrod), + asciigraph.Caption(eventsLegend), + ) + eventGraph := boxedView("Events Emitted", eventsPlot, graphWidth) + + eventSizePlot := asciigraph.Plot( + report.Audit.EventSizeBuffer.Data(graphWidth-15), + asciigraph.Height(graphHeight), + asciigraph.Width(graphWidth-15), + asciigraph.UpperBound(1), + ) + sizeGraph := boxedView("Event Sizes", eventSizePlot, graphWidth) + + graphStyle := lipgloss.NewStyle(). + Width(graphWidth). + Padding(0). + Margin(0). + Align(lipgloss.Left) + + return lipgloss.JoinVertical(lipgloss.Left, + graphStyle.Render( + eventGraph, + sizeGraph, + ), + ) +} + // tabView renders the tabbed content in the header. func tabView(selectedTab int) string { output := lipgloss.NewStyle(). @@ -520,5 +574,8 @@ var ( selectedColor = lipgloss.Color("4") - tabs = []string{"Common", "Backend", "Cache", "Watcher"} + failedEventStyle = lipgloss.NewStyle().Foreground(lipgloss.Color(fmt.Sprintf("%d", asciigraph.Red))) + trimmedEventStyle = lipgloss.NewStyle().Foreground(lipgloss.Color(fmt.Sprintf("%d", asciigraph.Goldenrod))) + + tabs = []string{"Common", "Backend", "Cache", "Watcher", "Audit"} ) diff --git a/tool/tctl/common/top/report.go b/tool/tctl/common/top/report.go index 3cdbabd2e2989..1c2981bf00ffd 100644 --- a/tool/tctl/common/top/report.go +++ b/tool/tctl/common/top/report.go @@ -58,6 +58,32 @@ type Report struct { Cluster ClusterStats // Watcher is watcher stats Watcher *WatcherStats + // Audit contains stats for audit event backends. + Audit *AuditStats +} + +// AuditStats contains metrics related to the audit log. +type AuditStats struct { + // FailedEventsCounter tallies the frequency of failed events. + FailedEventsCounter *Counter + // FailedEventsBuffer contains the historical frequencies of + // the FailedEventsCounter. + FailedEventsBuffer *utils.CircularBuffer + // EmittedEventsCounter tallies the frequency of all emitted events. + EmittedEventsCounter *Counter + // EmittedEventsBuffer contains the historical frequencies of + // the EmittedEventsCounter. + EmittedEventsBuffer *utils.CircularBuffer + // EventSizeCounter tallies the frequency of all events. + EventSizeCounter *Counter + // EventSizeBuffer contains the historical sizes of + // the EventSizeCounter. + EventSizeBuffer *utils.CircularBuffer + // EventsCounter tallies the frequency of trimmed events. + TrimmedEventsCounter *Counter + // TrimmedEventsBuffer contains the historical sizes of + // the TrimmedEventsCounter. + TrimmedEventsBuffer *utils.CircularBuffer } // WatcherStats contains watcher stats @@ -255,7 +281,8 @@ type Counter struct { } // SetFreq sets counter frequency based on the previous value -// and the time period +// and the time period. SetFreq should be preffered over UpdateFreq +// when initializing a Counter from previous statistics. func (c *Counter) SetFreq(prevCount Counter, period time.Duration) { if period == 0 { return @@ -264,6 +291,25 @@ func (c *Counter) SetFreq(prevCount Counter, period time.Duration) { c.Freq = &freq } +// UpdateFreq sets counter frequency based on the previous value +// and the time period. UpdateFreq should be preferred over SetFreq +// if the Counter is reused. +func (c *Counter) UpdateFreq(currentCount int64, period time.Duration) { + if period == 0 { + return + } + + // Do not calculate the frequency until there are two data points. + if c.Count == 0 && c.Freq == nil { + c.Count = currentCount + return + } + + freq := float64(currentCount-c.Count) / float64(period/time.Second) + c.Freq = &freq + c.Count = currentCount +} + // GetFreq returns frequency of the request func (c Counter) GetFreq() float64 { if c.Freq == nil { @@ -423,6 +469,13 @@ func generateReport(metrics map[string]*dto.MetricFamily, prev *Report, period t Roles: getGaugeValue(metrics[prometheus.BuildFQName(teleport.MetricNamespace, "", "roles_total")]), } + var auditStats *AuditStats + if prev != nil { + auditStats = prev.Audit + } + + re.Audit = getAuditStats(metrics, auditStats, period) + if prev != nil { re.Cluster.GenerateRequestsCount.SetFreq(prev.Cluster.GenerateRequestsCount, period) re.Cluster.GenerateRequestsThrottledCount.SetFreq(prev.Cluster.GenerateRequestsThrottledCount, period) @@ -547,6 +600,69 @@ func getWatcherStats(metrics map[string]*dto.MetricFamily, prev *WatcherStats, p return stats } +func getAuditStats(metrics map[string]*dto.MetricFamily, prev *AuditStats, period time.Duration) *AuditStats { + if prev == nil { + failed, err := utils.NewCircularBuffer(150) + if err != nil { + return nil + } + + events, err := utils.NewCircularBuffer(150) + if err != nil { + return nil + } + + trimmed, err := utils.NewCircularBuffer(150) + if err != nil { + return nil + } + + sizes, err := utils.NewCircularBuffer(150) + if err != nil { + return nil + } + + prev = &AuditStats{ + FailedEventsBuffer: failed, + FailedEventsCounter: &Counter{}, + EmittedEventsBuffer: events, + EmittedEventsCounter: &Counter{}, + TrimmedEventsBuffer: trimmed, + TrimmedEventsCounter: &Counter{}, + EventSizeBuffer: sizes, + EventSizeCounter: &Counter{}, + } + } + + updateCounter := func(metrics map[string]*dto.MetricFamily, metric string, counter *Counter, buf *utils.CircularBuffer) { + current := getCounterValue(metrics[metric]) + counter.UpdateFreq(current, period) + buf.Add(counter.GetFreq()) + } + + updateCounter(metrics, prometheus.BuildFQName("", "audit", "failed_emit_events"), prev.FailedEventsCounter, prev.FailedEventsBuffer) + updateCounter(metrics, prometheus.BuildFQName(teleport.MetricNamespace, "audit", "stored_trimmed_events"), prev.TrimmedEventsCounter, prev.TrimmedEventsBuffer) + + histogram := getHistogram(metrics[prometheus.BuildFQName(teleport.MetricNamespace, "", "audit_emitted_event_sizes")], atIndex(0)) + + prev.EmittedEventsCounter.UpdateFreq(histogram.Count, period) + prev.EmittedEventsBuffer.Add(prev.EmittedEventsCounter.GetFreq()) + + prev.EventSizeCounter.UpdateFreq(int64(histogram.Sum), period) + prev.EventSizeBuffer.Add(prev.EventSizeCounter.GetFreq()) + + return &AuditStats{ + FailedEventsBuffer: prev.FailedEventsBuffer, + FailedEventsCounter: prev.FailedEventsCounter, + EmittedEventsBuffer: prev.EmittedEventsBuffer, + EmittedEventsCounter: prev.EmittedEventsCounter, + TrimmedEventsBuffer: prev.TrimmedEventsBuffer, + TrimmedEventsCounter: prev.TrimmedEventsCounter, + EventSizeBuffer: prev.EventSizeBuffer, + EventSizeCounter: prev.EventSizeCounter, + } +} + func getRemoteClusters(metric *dto.MetricFamily) []RemoteCluster { if metric == nil || metric.GetType() != dto.MetricType_GAUGE || len(metric.Metric) == 0 { return nil diff --git a/tool/tsh/common/latency.go b/tool/tsh/common/latency.go index 6089f58c7bd44..6c73bd73d623b 100644 --- a/tool/tsh/common/latency.go +++ b/tool/tsh/common/latency.go @@ -151,6 +151,7 @@ func (m *latencyModel) View() string { [][]float64{clientData, serverData}, asciigraph.Height(m.h-4), asciigraph.Width(m.w), + asciigraph.UpperBound(1), asciigraph.SeriesColors(clientColor, serverColor), asciigraph.Caption(legend), )