From 755d3e1b0610cf8df6da22ecd29473d080c28ef9 Mon Sep 17 00:00:00 2001 From: Kevin Hoffman Date: Fri, 12 Apr 2024 09:54:11 -0400 Subject: [PATCH 1/2] makes uptimes and runtimes real --- internal/agent-api/client.go | 17 +++++++++++++++++ internal/node/controlapi.go | 34 ---------------------------------- internal/node/workload_mgr.go | 29 +++++++++++++++++++++++++++-- nex/nodes.go | 2 +- 4 files changed, 45 insertions(+), 37 deletions(-) diff --git a/internal/agent-api/client.go b/internal/agent-api/client.go index da927f51..f8919e6f 100644 --- a/internal/agent-api/client.go +++ b/internal/agent-api/client.go @@ -38,6 +38,9 @@ type AgentClient struct { eventReceived EventCallback logReceived LogCallback + execTotalNanos int64 + workloadStartedAt time.Time + subz []*nats.Subscription } @@ -125,6 +128,7 @@ func (a *AgentClient) DeployWorkload(request *DeployRequest) (*DeployResponse, e a.log.Error("Failed to deserialize deployment response", slog.Any("error", err)) return nil, err } + a.workloadStartedAt = time.Now().UTC() return &deployResponse, nil } @@ -162,6 +166,19 @@ func (a *AgentClient) Undeploy() error { return nil } +func (a *AgentClient) RecordExecTime(elapsedNanos int64) { + a.execTotalNanos += elapsedNanos +} + +func (a *AgentClient) ExecTimeNanos() int64 { + return a.execTotalNanos +} + +// Returns the time difference between now and when the agent started +func (a *AgentClient) UptimeMillis() time.Duration { + return time.Since(a.workloadStartedAt) +} + func (a *AgentClient) RunTrigger(ctx context.Context, tracer trace.Tracer, subject string, data []byte) (*nats.Msg, error) { intmsg := nats.NewMsg(fmt.Sprintf("agentint.%s.trigger", a.agentID)) // TODO: inject tracer context into message header diff --git a/internal/node/controlapi.go b/internal/node/controlapi.go index 8f59b315..2d2d23ba 100644 --- a/internal/node/controlapi.go +++ b/internal/node/controlapi.go @@ -353,40 +353,6 @@ func summarizeMachines(workloads []controlapi.MachineSummary, namespace string) return machines } -// func summarizeMachines(vms *map[string]*runningFirecracker, namespace string) []controlapi.MachineSummary { -// machines := make([]controlapi.MachineSummary, 0) -// now := time.Now().UTC() -// for _, v := range *vms { -// if v.namespace == namespace { -// var desc string -// if v.deployRequest.Description != nil { -// desc = *v.deployRequest.Description // FIXME-- audit controlapi.WorkloadSummary -// } - -// var workloadType string -// if v.deployRequest.WorkloadType != nil { -// workloadType = *v.deployRequest.WorkloadType -// } - -// machine := controlapi.MachineSummary{ -// Id: v.vmmID, -// Healthy: true, // TODO cache last health status -// Uptime: myUptime(now.Sub(v.machineStarted)), -// Workload: controlapi.WorkloadSummary{ -// Name: v.deployRequest.DecodedClaims.Subject, -// Description: desc, -// Runtime: myUptime(now.Sub(v.workloadStarted)), -// WorkloadType: workloadType, -// //Hash: v.deployedWorkload.DecodedClaims.Data["hash"].(string), -// }, -// } - -// machines = append(machines, machine) -// } -// } -// return machines -// } - func validateIssuer(issuer string, validIssuers []string) bool { if len(validIssuers) == 0 { return true diff --git a/internal/node/workload_mgr.go b/internal/node/workload_mgr.go index 1426b4e8..81cc6251 100644 --- a/internal/node/workload_mgr.go +++ b/internal/node/workload_mgr.go @@ -293,15 +293,39 @@ func (w *WorkloadManager) RunningWorkloads() ([]controlapi.MachineSummary, error summaries := make([]controlapi.MachineSummary, len(procs)) for i, p := range procs { + uptimeFriendly := "unknown" + runtimeFriendly := "unknown" + agentClient, ok := w.activeAgents[p.ID] + if ok { + uptimeFriendly = myUptime(agentClient.UptimeMillis()) + if *p.DeployRequest.WorkloadType == "v8" || *p.DeployRequest.WorkloadType == "wasm" { + nanoTime := fmt.Sprintf("%dns", agentClient.ExecTimeNanos()) + rt, err := time.ParseDuration(nanoTime) + if err == nil { + if rt.Nanoseconds() < 1000 { + runtimeFriendly = nanoTime + } else if rt.Milliseconds() < 1000 { + runtimeFriendly = fmt.Sprintf("%dms", rt.Milliseconds()) + } else { + runtimeFriendly = myUptime(rt) + } + } else { + w.log.Warn("Failed to generate parsed time from nanos", slog.Any("error", err)) + } + } else { + runtimeFriendly = uptimeFriendly + } + } + summaries[i] = controlapi.MachineSummary{ Id: p.ID, Healthy: true, - Uptime: "TBD", + Uptime: uptimeFriendly, Namespace: p.Namespace, Workload: controlapi.WorkloadSummary{ Name: p.Name, Description: *p.DeployRequest.WorkloadName, - Runtime: "TBD", // TODO: replace with function exec time OR service uptime + Runtime: runtimeFriendly, WorkloadType: *p.DeployRequest.WorkloadType, Hash: p.DeployRequest.Hash, }, @@ -479,6 +503,7 @@ func (w *WorkloadManager) generateTriggerHandler(workloadID string, tsub string, w.log.Warn("failed to log function runtime", slog.Any("err", err)) } _ = w.publishFunctionExecSucceeded(workloadID, tsub, runTimeNs64) + agentClient.RecordExecTime(runTimeNs64) parentSpan.AddEvent("published success event") w.t.FunctionTriggers.Add(w.ctx, 1) diff --git a/nex/nodes.go b/nex/nodes.go index 0b376cd5..fd5d5e6e 100644 --- a/nex/nodes.go +++ b/nex/nodes.go @@ -86,7 +86,7 @@ func renderNodeInfo(info *controlapi.InfoResponse, id string) { cols.Println() cols.AddRow("Id", m.Id) cols.AddRow("Healthy", m.Healthy) - cols.AddRow("Runtime", m.Uptime) + cols.AddRow("Runtime", m.Workload.Runtime) cols.AddRow("Name", m.Workload.Name) cols.AddRow("Description", m.Workload.Description) } From 9d042cacf31ee73812bfed7e1c7776994b563347 Mon Sep 17 00:00:00 2001 From: Kevin Hoffman Date: Fri, 12 Apr 2024 10:08:26 -0400 Subject: [PATCH 2/2] atomic increment nanos --- internal/agent-api/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/agent-api/client.go b/internal/agent-api/client.go index f8919e6f..9761f52a 100644 --- a/internal/agent-api/client.go +++ b/internal/agent-api/client.go @@ -167,7 +167,7 @@ func (a *AgentClient) Undeploy() error { } func (a *AgentClient) RecordExecTime(elapsedNanos int64) { - a.execTotalNanos += elapsedNanos + atomic.AddInt64(&a.execTotalNanos, elapsedNanos) } func (a *AgentClient) ExecTimeNanos() int64 {