[v15] Add Audit Log tab to tctl top

tool/tctl/common/top/model.go

@@ -103,6 +103,8 @@ func (m *topModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 			m.selected = 2
 		case "4":
 			m.selected = 3
+		case "5":
+			m.selected = 4
 		case "right":
 			m.selected = min(m.selected+1, len(tabs)-1)
 		case "left":
@@ -218,6 +220,8 @@ func (m *topModel) contentView() string {
 		return renderCache(m.report, m.height, m.width)
 	case 3:
 		return renderWatcher(m.report, m.height, m.width)
+	case 4:
+		return renderAudit(m.report, m.height, m.width)
 	default:
 		return ""
 	}
@@ -412,6 +416,7 @@ func renderWatcher(report *Report, height, width int) string {
 		eventData,
 		asciigraph.Height(graphHeight),
 		asciigraph.Width(graphWidth-15),
+		asciigraph.UpperBound(1),
 	)
 	eventCountContent := boxedView("Events/Sec", countPlot, graphWidth)
 
@@ -423,6 +428,7 @@ func renderWatcher(report *Report, height, width int) string {
 		sizeData,
 		asciigraph.Height(graphHeight),
 		asciigraph.Width(graphWidth-15),
+		asciigraph.UpperBound(1),
 	)
 	eventSizeContent := boxedView("Bytes/Sec", sizePlot, graphWidth)
 
@@ -449,6 +455,54 @@ func renderWatcher(report *Report, height, width int) string {
 	)
 }
 
+// renderAudit generates the view for the audit stats tab.
+func renderAudit(report *Report, height, width int) string {
+	graphHeight := height / 3
+	graphWidth := width
+
+	eventsLegend := lipgloss.JoinHorizontal(
+		lipgloss.Left,
+		"- Emitted",
+		failedEventStyle.Render(" - Failed"),
+		trimmedEventStyle.Render(" - Trimmed"),
+	)
+
+	eventsPlot := asciigraph.PlotMany(
+		[][]float64{
+			report.Audit.EmittedEventsBuffer.Data(graphWidth - 15),
+			report.Audit.FailedEventsBuffer.Data(graphWidth - 15),
+			report.Audit.TrimmedEventsBuffer.Data(graphWidth - 15),
+		},
+		asciigraph.Height(graphHeight),
+		asciigraph.Width(graphWidth-15),
+		asciigraph.UpperBound(1),
+		asciigraph.SeriesColors(asciigraph.Default, asciigraph.Red, asciigraph.Goldenrod),
+		asciigraph.Caption(eventsLegend),
+	)
+	eventGraph := boxedView("Events Emitted", eventsPlot, graphWidth)
+
+	eventSizePlot := asciigraph.Plot(
+		report.Audit.EventSizeBuffer.Data(graphWidth-15),
+		asciigraph.Height(graphHeight),
+		asciigraph.Width(graphWidth-15),
+		asciigraph.UpperBound(1),
+	)
+	sizeGraph := boxedView("Event Sizes", eventSizePlot, graphWidth)
+
+	graphStyle := lipgloss.NewStyle().
+		Width(graphWidth).
+		Padding(0).
+		Margin(0).
+		Align(lipgloss.Left)
+
+	return lipgloss.JoinVertical(lipgloss.Left,
+		graphStyle.Render(
+			eventGraph,
+			sizeGraph,
+		),
+	)
+}
+
 // tabView renders the tabbed content in the header.
 func tabView(selectedTab int) string {
 	output := lipgloss.NewStyle().
@@ -520,5 +574,8 @@ var (
 
 	selectedColor = lipgloss.Color("4")
 
-	tabs = []string{"Common", "Backend", "Cache", "Watcher"}
+	failedEventStyle  = lipgloss.NewStyle().Foreground(lipgloss.Color(fmt.Sprintf("%d", asciigraph.Red)))
+	trimmedEventStyle = lipgloss.NewStyle().Foreground(lipgloss.Color(fmt.Sprintf("%d", asciigraph.Goldenrod)))
+
+	tabs = []string{"Common", "Backend", "Cache", "Watcher", "Audit"}
 )

tool/tctl/common/top/report.go

@@ -58,6 +58,32 @@ type Report struct {
 	Cluster ClusterStats
 	// Watcher is watcher stats
 	Watcher *WatcherStats
+	// Audit contains stats for audit event backends.
+	Audit *AuditStats
+}
+
+// AuditStats contains metrics related to the audit log.
+type AuditStats struct {
+	// FailedEventsCounter tallies the frequency of failed events.
+	FailedEventsCounter *Counter
+	// FailedEventsBuffer contains the historical frequencies of
+	// the FailedEventsCounter.
+	FailedEventsBuffer *utils.CircularBuffer
+	// EmittedEventsCounter tallies the frequency of all emitted events.
+	EmittedEventsCounter *Counter
+	// EmittedEventsBuffer contains the historical frequencies of
+	// the EmittedEventsCounter.
+	EmittedEventsBuffer *utils.CircularBuffer
+	// EventSizeCounter tallies the frequency of all events.
+	EventSizeCounter *Counter
+	// EventSizeBuffer contains the historical sizes of
+	// the EventSizeCounter.
+	EventSizeBuffer *utils.CircularBuffer
+	// EventsCounter tallies the frequency of trimmed events.
+	TrimmedEventsCounter *Counter
+	// TrimmedEventsBuffer contains the historical sizes of
+	// the TrimmedEventsCounter.
+	TrimmedEventsBuffer *utils.CircularBuffer
 }
 
 // WatcherStats contains watcher stats
@@ -255,7 +281,8 @@ type Counter struct {
 }
 
 // SetFreq sets counter frequency based on the previous value
-// and the time period
+// and the time period. SetFreq should be preffered over UpdateFreq
+// when initializing a Counter from previous statistics.
 func (c *Counter) SetFreq(prevCount Counter, period time.Duration) {
 	if period == 0 {
 		return
@@ -264,6 +291,25 @@ func (c *Counter) SetFreq(prevCount Counter, period time.Duration) {
 	c.Freq = &freq
 }
 
+// UpdateFreq sets counter frequency based on the previous value
+// and the time period. UpdateFreq should be preferred over SetFreq
+// if the Counter is reused.
+func (c *Counter) UpdateFreq(currentCount int64, period time.Duration) {
+	if period == 0 {
+		return
+	}
+
+	// Do not calculate the frequency until there are two data points.
+	if c.Count == 0 && c.Freq == nil {
+		c.Count = currentCount
+		return
+	}
+
+	freq := float64(currentCount-c.Count) / float64(period/time.Second)
+	c.Freq = &freq
+	c.Count = currentCount
+}
+
 // GetFreq returns frequency of the request
 func (c Counter) GetFreq() float64 {
 	if c.Freq == nil {
@@ -423,6 +469,13 @@ func generateReport(metrics map[string]*dto.MetricFamily, prev *Report, period t
 		Roles:                          getGaugeValue(metrics[prometheus.BuildFQName(teleport.MetricNamespace, "", "roles_total")]),
 	}
 
+	var auditStats *AuditStats
+	if prev != nil {
+		auditStats = prev.Audit
+	}
+
+	re.Audit = getAuditStats(metrics, auditStats, period)
+
 	if prev != nil {
 		re.Cluster.GenerateRequestsCount.SetFreq(prev.Cluster.GenerateRequestsCount, period)
 		re.Cluster.GenerateRequestsThrottledCount.SetFreq(prev.Cluster.GenerateRequestsThrottledCount, period)
@@ -547,6 +600,69 @@ func getWatcherStats(metrics map[string]*dto.MetricFamily, prev *WatcherStats, p
 	return stats
 }
 
+func getAuditStats(metrics map[string]*dto.MetricFamily, prev *AuditStats, period time.Duration) *AuditStats {
+	if prev == nil {
+		failed, err := utils.NewCircularBuffer(150)
+		if err != nil {
+			return nil
+		}
+
+		events, err := utils.NewCircularBuffer(150)
+		if err != nil {
+			return nil
+		}
+
+		trimmed, err := utils.NewCircularBuffer(150)
+		if err != nil {
+			return nil
+		}
+
+		sizes, err := utils.NewCircularBuffer(150)
+		if err != nil {
+			return nil
+		}
+
+		prev = &AuditStats{
+			FailedEventsBuffer:   failed,
+			FailedEventsCounter:  &Counter{},
+			EmittedEventsBuffer:  events,
+			EmittedEventsCounter: &Counter{},
+			TrimmedEventsBuffer:  trimmed,
+			TrimmedEventsCounter: &Counter{},
+			EventSizeBuffer:      sizes,
+			EventSizeCounter:     &Counter{},
+		}
+	}
+
+	updateCounter := func(metrics map[string]*dto.MetricFamily, metric string, counter *Counter, buf *utils.CircularBuffer) {
+		current := getCounterValue(metrics[metric])
+		counter.UpdateFreq(current, period)
+		buf.Add(counter.GetFreq())
+	}
+
+	updateCounter(metrics, prometheus.BuildFQName("", "audit", "failed_emit_events"), prev.FailedEventsCounter, prev.FailedEventsBuffer)
+	updateCounter(metrics, prometheus.BuildFQName(teleport.MetricNamespace, "audit", "stored_trimmed_events"), prev.TrimmedEventsCounter, prev.TrimmedEventsBuffer)
+
+	histogram := getHistogram(metrics[prometheus.BuildFQName(teleport.MetricNamespace, "", "audit_emitted_event_sizes")], atIndex(0))
+
+	prev.EmittedEventsCounter.UpdateFreq(histogram.Count, period)
+	prev.EmittedEventsBuffer.Add(prev.EmittedEventsCounter.GetFreq())
+
+	prev.EventSizeCounter.UpdateFreq(int64(histogram.Sum), period)
+	prev.EventSizeBuffer.Add(prev.EventSizeCounter.GetFreq())
+
+	return &AuditStats{
+		FailedEventsBuffer:   prev.FailedEventsBuffer,
+		FailedEventsCounter:  prev.FailedEventsCounter,
+		EmittedEventsBuffer:  prev.EmittedEventsBuffer,
+		EmittedEventsCounter: prev.EmittedEventsCounter,
+		TrimmedEventsBuffer:  prev.TrimmedEventsBuffer,
+		TrimmedEventsCounter: prev.TrimmedEventsCounter,
+		EventSizeBuffer:      prev.EventSizeBuffer,
+		EventSizeCounter:     prev.EventSizeCounter,
+	}
+}
+
 func getRemoteClusters(metric *dto.MetricFamily) []RemoteCluster {
 	if metric == nil || metric.GetType() != dto.MetricType_GAUGE || len(metric.Metric) == 0 {
 		return nil

tool/tsh/common/latency.go

@@ -151,6 +151,7 @@ func (m *latencyModel) View() string {
 		[][]float64{clientData, serverData},
 		asciigraph.Height(m.h-4),
 		asciigraph.Width(m.w),
+		asciigraph.UpperBound(1),
 		asciigraph.SeriesColors(clientColor, serverColor),
 		asciigraph.Caption(legend),
 	)