feat: added progress display during discovery

* feat: added progress display during discovery
* feat: moved log to own library (prep for future improvement)
* update discovery screenshot to illustrate progress during discovery

cmd/analysis.go

@@ -11,7 +11,7 @@ import (
 	"sort"
 	"strings"
 
-	log "github.com/sirupsen/logrus"
+	"opsani-ignite/log"
 
 	appmodel "opsani-ignite/app/model"
 	opsmath "opsani-ignite/math"

cmd/ignite.go

@@ -12,11 +12,11 @@ import (
 	"sort"
 	"time"
 
-	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
 	appmodel "opsani-ignite/app/model"
 
+	"opsani-ignite/log"
 	prom "opsani-ignite/sources/prometheus"
 )
 
@@ -53,16 +53,6 @@ func isQualifiedApp(app *appmodel.App) bool {
 	return app.Analysis.Rating >= 0
 }
 
-func setupLogLevel() {
-	if showDebug {
-		log.SetLevel(log.TraceLevel)
-	} else if suppressWarnings {
-		log.SetLevel(log.ErrorLevel)
-	} else {
-		log.SetLevel(log.InfoLevel)
-	}
-}
-
 func displayConfig(namespace, deployment string) {
 	msgs := make([]string, 0)
 
@@ -134,7 +124,7 @@ func runIgnite(cmd *cobra.Command, args []string) {
 	}
 	defer logFile.Close()
 	log.SetOutput(logFile)
-	setupLogLevel()
+	log.SetupLogLevel(showDebug, suppressWarnings)
 
 	// determine namespace & deployment selection
 	namespace := ""
@@ -152,7 +142,12 @@ func runIgnite(cmd *cobra.Command, args []string) {
 
 	// get applications from the cluster
 	prom.Init()
-	apps, err := prom.PromGetAll(ctx, promUri, namespace, deployment, "apps/v1", "Deployment", timeStart, timeEnd, timeStep)
+	apps := make([]*appmodel.App, 0)
+	err = log.GoWithProgress(func (progressCallback log.ProgressUpdateFunc) error {
+		var innerErr error
+		apps, innerErr = prom.PromGetAll(ctx, promUri, namespace, deployment, "apps/v1", "Deployment", timeStart, timeEnd, timeStep, progressCallback)
+		return innerErr
+	})
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Failed to obtain data from Prometheus at %q: %v", promUri, err)
 		os.Exit(1)

cmd/output.go

@@ -12,8 +12,9 @@ import (
 	"sort"
 	"strings"
 
+	"opsani-ignite/log"
+
 	"github.com/olekukonko/tablewriter"
-	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
 
 	appmodel "opsani-ignite/app/model"

log/log.go

@@ -0,0 +1,88 @@
+/*
+Copyright © 2021 Opsani <[email protected]>
+This file is part of https://github.com/opsani/opsani-ignite
+*/
+
+package log
+
+import (
+	"io"
+
+	"github.com/sirupsen/logrus"
+)
+
+// SetOutput sets the standard logger output.
+func SetOutput(out io.Writer) {
+	logrus.SetOutput(out)
+}
+
+// SetupLogLevel sets up the standard logger level in accordance with command line options.
+func SetupLogLevel(showDebug bool, suppressWarnings bool) {
+	if showDebug {
+		logrus.SetLevel(logrus.TraceLevel)
+	} else if suppressWarnings {
+		logrus.SetLevel(logrus.ErrorLevel)
+	} else {
+		logrus.SetLevel(logrus.InfoLevel)
+	}
+}
+
+// Trace logs a message at level Trace on the standard logger.
+func Trace(args ...interface{}) {
+	logrus.Trace(args...)
+}
+
+// Info logs a message at level Info on the standard logger.
+func Info(args ...interface{}) {
+	logrus.Info(args...)
+}
+
+// Print logs a message at level Info on the standard logger.
+func Print(args ...interface{}) {
+	logrus.Print(args...)
+}
+
+// Warn logs a message at level Warning on the standard logger.
+func Warn(args ...interface{}) {
+	logrus.Warn(args...)
+}
+
+// Error logs a message at level Error on the standard logger.
+func Error(args ...interface{}) {
+	logrus.Error(args...)
+}
+
+// Fatal logs a message at level Fatal on the standard logger.
+func Fatal(args ...interface{}) {
+	logrus.Fatal(args...)
+}
+
+// Tracef logs a message at level Trace on the standard logger.
+func Tracef(format string, args ...interface{}) {
+	logrus.Tracef(format, args...)
+}
+
+// Infof logs a message at level Info on the standard logger.
+func Infof(format string, args ...interface{}) {
+	logrus.Infof(format, args...)
+}
+
+// Printf logs a message at level Info on the standard logger.
+func Printf(format string, args ...interface{}) {
+	logrus.Printf(format, args...)
+}
+
+// Warnf logs a message at level Warning on the standard logger.
+func Warnf(format string, args ...interface{}) {
+	logrus.Warnf(format, args...)
+}
+
+// Errorf logs a message at level Error on the standard logger.
+func Errorf(format string, args ...interface{}) {
+	logrus.Errorf(format, args...)
+}
+
+// Fatalf logs a message at level Fatal on the standard logger.
+func Fatalf(format string, args ...interface{}) {
+	logrus.Fatalf(format, args...)
+}

log/progress.go

@@ -0,0 +1,94 @@
+/*
+Copyright © 2021 Opsani <[email protected]>
+This file is part of https://github.com/opsani/opsani-ignite
+*/
+
+package log
+
+import (
+	"fmt"
+	"math"
+	"os"
+	"sync"
+	"time"
+)
+
+// ProgressInfo holds the values used to show progress
+type ProgressInfo struct {
+	NamespacesTotal int
+	NamespacesDone  int
+	WorkloadsTotal  int
+	WorkloadsDone   int
+}
+
+// ProgressUpdateFunc is the signature of the progress update callback function.
+// The update may set absolute or relative values (relative values are +=).
+type ProgressUpdateFunc func(info ProgressInfo, relative bool)
+
+// RunnerFunc is the signature for the function to run with GoWithProgress
+type RunnerFunc func(infoCallback ProgressUpdateFunc) error
+
+type progressState struct {
+	lock sync.Mutex
+	info ProgressInfo
+}
+
+func (s *progressState) updateInfo(info ProgressInfo, relative bool) {
+	s.lock.Lock()
+	if relative {
+		s.info.NamespacesTotal += info.NamespacesTotal
+		s.info.NamespacesDone  += info.NamespacesDone
+		s.info.WorkloadsTotal  += info.WorkloadsTotal
+		s.info.WorkloadsDone   += info.WorkloadsDone
+	} else {
+		s.info = info
+	}
+	s.lock.Unlock()
+}
+
+func (s *progressState) renderProgress(startTime time.Time, final bool) {
+	// safely grab a copy of the info
+	s.lock.Lock()
+	info := s.info
+	s.lock.Unlock()
+
+	// display info as progress
+	now := time.Now()
+	elapsed := math.Round(now.Sub(startTime).Seconds()*10) / 10
+	fmt.Fprintf(os.Stderr, "\rCollecting data (%.1fs): %v of %v namespace(s) and %v of %v application(s) completed... ",
+		elapsed, info.NamespacesDone, info.NamespacesTotal, info.WorkloadsDone, info.WorkloadsTotal)
+	if final {
+		fmt.Fprintf(os.Stderr, "done.\n\n")
+	}
+}
+
+// GoWithProgress executes the given runner function as a goroutine while showing progress
+func GoWithProgress(runner RunnerFunc) error {
+	done := make(chan error)
+	state := progressState{}
+	state.renderProgress(time.Now(), false)
+
+	// run the runner function and notify when done
+	go func() {
+		err := runner(func(info ProgressInfo, relative bool) { state.updateInfo(info, relative) })
+		done <- err
+	}()
+
+	startTime := time.Now()
+	var runnerError error
+loop:
+	for {
+		select {
+		case err := <-done:
+			runnerError = err
+			break loop
+		default:
+			state.renderProgress(startTime, false)
+			time.Sleep(time.Duration(100 * time.Millisecond))
+		}
+	}
+	close(done)
+	state.renderProgress(startTime, true)
+
+	return runnerError
+}

log/progress_demo.go

@@ -0,0 +1,61 @@
+/*
+Copyright © 2021 Opsani <[email protected]>
+This file is part of https://github.com/opsani/opsani-ignite
+*/
+
+package log
+
+import (
+	"time"
+)
+
+func job_abs(update ProgressUpdateFunc) error {
+	apps := 0
+	for n := 0; n < 5; n++ {
+		for a := 0; a < 7; a++ {
+			update(ProgressInfo{
+				NamespacesTotal: 5,
+				NamespacesDone:  n,
+				WorkloadsTotal:  (n + 1) * 7,
+				WorkloadsDone:   apps,
+			}, false)
+			apps++
+			time.Sleep(325 * time.Millisecond)
+		}
+	}
+	update(ProgressInfo{
+		NamespacesTotal: 5,
+		NamespacesDone:  5,
+		WorkloadsTotal:  5 * 7,
+		WorkloadsDone:   apps,
+	}, false)
+	return nil
+}
+
+func job_rel(update ProgressUpdateFunc) error {
+	update(ProgressInfo{
+		NamespacesTotal: 5,
+		NamespacesDone:  0,
+		WorkloadsTotal:  5 * 7,
+		WorkloadsDone:   0,
+	}, false)
+	for n := 0; n < 5; n++ {
+		for a := 0; a < 7; a++ {
+			update(ProgressInfo{WorkloadsDone: 1}, true)
+			time.Sleep(325 * time.Millisecond)
+		}
+		update(ProgressInfo{NamespacesDone: 1}, true)
+	}
+	return nil
+}
+
+
+func DemoProgress() error {
+	if err := GoWithProgress(job_abs); err != nil {
+		return err
+	}
+	if err := GoWithProgress(job_rel); err != nil {
+		return err
+	}
+	return nil
+}

sources/prometheus/containers.go

@@ -9,16 +9,17 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	appmodel "opsani-ignite/app/model"
-	opsmath "opsani-ignite/math"
 	"reflect"
 	"strings"
 	"text/template"
 	"time"
 
 	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	"github.com/prometheus/common/model"
-	log "github.com/sirupsen/logrus"
+
+	appmodel "opsani-ignite/app/model"
+	"opsani-ignite/log"
+	opsmath "opsani-ignite/math"
 )
 
 // get request or limit values for all resources of all containers of the specificed application
@@ -137,7 +138,7 @@ func getContainersUseValueMap(ctx context.Context, promApi v1.API, app *appmodel
 	valueMap := make(map[string]float64, len(app.Containers))
 	for _, c := range series { // c is *model.SampleStream
 		if len(c.Metric) > 1 {
-			log.Warningf("metrics returned for query %q contain labels %v, expected %v, ignoring extras (app %v)", query, c.Metric, []string{"container"}, app.Metadata)
+			log.Warnf("metrics returned for query %q contain labels %v, expected %v, ignoring extras (app %v)", query, c.Metric, []string{"container"}, app.Metadata)
 		}
 		if len(c.Metric) == 0 {
 			//log.Tracef("(expected) skipping metrics for without labels for app %v, query %q", app.Metadata, query)