feat: grafloki start/stop

cli/cli/command_framework/highlevel/engine_consuming_kurtosis_command/engine_consuming_kurtosis_command.go

@@ -187,7 +187,7 @@ func (cmd *EngineConsumingKurtosisCommand) getSetupFunc() func(context.Context)
 		kurtosisBackend := engineManager.GetKurtosisBackend()
 
 		dontRestartAPIContainers := false
-		engineClient, closeClientFunc, err := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain, defaults.DefaultLogRetentionPeriod)
+		engineClient, closeClientFunc, err := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain, defaults.DefaultLogRetentionPeriod, defaults.DefaultLogsSinks)
 		if err != nil {
 			return nil, stacktrace.Propagate(err, "An error occurred creating a new Kurtosis engine client")
 		}

cli/cli/command_str_consts/command_str_consts.go

@@ -81,6 +81,9 @@ const (
 	GitHubLogoutCmdStr      = "logout"
 	GitHubTokenCmdStr       = "token"
 	GitHubStatusCmdStr      = "status"
+	GraflokiCmdStr          = "grafloki"
+	GraflokiStartCmdStr     = "start"
+	GraflokiStopCmdStr      = "stop"
 )
 
 // TODO: added constant error message here, can we move to another file later.

cli/cli/commands/cluster/set/set.go

@@ -124,7 +124,7 @@ func run(ctx context.Context, flags *flags.ParsedFlags, args *args.ParsedArgs) e
 	// TODO - fix the idempotent starter longer term
 	if engineStatus == engine_manager.EngineStatus_Stopped {
 		dontRestartAPIContainers := false
-		_, engineClientCloseFunc, err := engineManagerNewCluster.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain, defaults.DefaultLogRetentionPeriod)
+		_, engineClientCloseFunc, err := engineManagerNewCluster.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain, defaults.DefaultLogRetentionPeriod, defaults.DefaultLogsSinks)
 		if err != nil {
 			return stacktrace.Propagate(err, "Engine could not be started after cluster was updated. Its status can be retrieved "+
 				"running 'kurtosis %s %s' and it can potentially be started running 'kurtosis %s %s'",

cli/cli/commands/enclave/add/add.go

@@ -121,7 +121,7 @@ func run(
 		return stacktrace.Propagate(err, "An error occurred creating an engine manager.")
 	}
 
-	engineClient, closeClientFunc, err := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain, defaults.DefaultLogRetentionPeriod)
+	engineClient, closeClientFunc, err := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, defaults.DefaultEngineLogLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain, defaults.DefaultLogRetentionPeriod, defaults.DefaultLogsSinks)
 	if err != nil {
 		return stacktrace.Propagate(err, "An error occurred creating a new Kurtosis engine client")
 	}

cli/cli/commands/engine/restart/restart.go

@@ -178,7 +178,7 @@ func run(_ context.Context, flags *flags.ParsedFlags, _ *args.ParsedArgs) error
 
 	var engineClientCloseFunc func() error
 	var restartEngineErr error
-	_, engineClientCloseFunc, restartEngineErr = engineManager.RestartEngineIdempotently(ctx, logLevel, engineVersion, restartEngineOnSameVersionIfAnyRunning, enclavePoolSize, shouldStartInDebugMode, githubAuthTokenOverride, shouldRestartAPIContainers, domain, logRetentionPeriodStr)
+	_, engineClientCloseFunc, restartEngineErr = engineManager.RestartEngineIdempotently(ctx, logLevel, engineVersion, restartEngineOnSameVersionIfAnyRunning, enclavePoolSize, shouldStartInDebugMode, githubAuthTokenOverride, shouldRestartAPIContainers, domain, logRetentionPeriodStr, defaults.DefaultLogsSinks)
 	if restartEngineErr != nil {
 		return stacktrace.Propagate(restartEngineErr, "An error occurred restarting the Kurtosis engine")
 	}

cli/cli/commands/engine/start/start.go

@@ -175,13 +175,13 @@ func run(_ context.Context, flags *flags.ParsedFlags, _ *args.ParsedArgs) error
 	if engineVersion == defaultEngineVersion && isDebugMode {
 		engineDebugVersion := fmt.Sprintf("%s-%s", kurtosis_version.KurtosisVersion, defaults.DefaultKurtosisContainerDebugImageNameSuffix)
 		logrus.Infof("Starting Kurtosis engine in debug mode from image '%v%v%v'...", kurtosisTechEngineImagePrefix, imageVersionDelimiter, engineDebugVersion)
-		_, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithCustomVersion(ctx, engineDebugVersion, logLevel, enclavePoolSize, true, githubAuthTokenOverride, shouldRestartAPIContainers, domain, logRetentionPeriodStr)
+		_, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithCustomVersion(ctx, engineDebugVersion, logLevel, enclavePoolSize, true, githubAuthTokenOverride, shouldRestartAPIContainers, domain, logRetentionPeriodStr, defaults.DefaultLogsSinks)
 	} else if engineVersion == defaultEngineVersion {
 		logrus.Infof("Starting Kurtosis engine from image '%v%v%v'...", kurtosisTechEngineImagePrefix, imageVersionDelimiter, kurtosis_version.KurtosisVersion)
-		_, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, logLevel, enclavePoolSize, githubAuthTokenOverride, shouldRestartAPIContainers, domain, logRetentionPeriodStr)
+		_, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, logLevel, enclavePoolSize, githubAuthTokenOverride, shouldRestartAPIContainers, domain, logRetentionPeriodStr, defaults.DefaultLogsSinks)
 	} else {
 		logrus.Infof("Starting Kurtosis engine from image '%v%v%v'...", kurtosisTechEngineImagePrefix, imageVersionDelimiter, engineVersion)
-		_, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithCustomVersion(ctx, engineVersion, logLevel, enclavePoolSize, defaults.DefaultEnableDebugMode, githubAuthTokenOverride, shouldRestartAPIContainers, domain, logRetentionPeriodStr)
+		_, engineClientCloseFunc, startEngineErr = engineManager.StartEngineIdempotentlyWithCustomVersion(ctx, engineVersion, logLevel, enclavePoolSize, defaults.DefaultEnableDebugMode, githubAuthTokenOverride, shouldRestartAPIContainers, domain, logRetentionPeriodStr, defaults.DefaultLogsSinks)
 	}
 	if startEngineErr != nil {
 		return stacktrace.Propagate(startEngineErr, "An error occurred starting the Kurtosis engine")

cli/cli/commands/github/login/login.go

@@ -71,7 +71,7 @@ func RestartEngineAfterGitHubAuth(ctx context.Context) error {
 	var engineClientCloseFunc func() error
 	var restartEngineErr error
 	dontRestartAPIContainers := false
-	_, engineClientCloseFunc, restartEngineErr = engineManager.RestartEngineIdempotently(ctx, defaults.DefaultEngineLogLevel, defaultEngineVersion, restartEngineOnSameVersionIfAnyRunning, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultEnableDebugMode, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain, defaults.DefaultLogRetentionPeriod)
+	_, engineClientCloseFunc, restartEngineErr = engineManager.RestartEngineIdempotently(ctx, defaults.DefaultEngineLogLevel, defaultEngineVersion, restartEngineOnSameVersionIfAnyRunning, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultEnableDebugMode, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain, defaults.DefaultLogRetentionPeriod, defaults.DefaultLogsSinks)
 	if restartEngineErr != nil {
 		return stacktrace.Propagate(restartEngineErr, "An error occurred restarting the Kurtosis engine")
 	}

cli/cli/commands/grafloki/grafloki.go

@@ -0,0 +1,21 @@
+package grafloki
+
+import (
+	"github.com/kurtosis-tech/kurtosis/cli/cli/command_str_consts"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/commands/grafloki/start"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/commands/grafloki/stop"
+	"github.com/spf13/cobra"
+)
+
+// GraflokiCmd suppressing exhaustruct requirement because this struct has ~40 properties
+// nolint: exhaustruct
+var GraflokiCmd = &cobra.Command{
+	Use:   command_str_consts.GraflokiCmdStr,
+	Short: "Start Grafana/Loki for log collection",
+	RunE:  nil,
+}
+
+func init() {
+	GraflokiCmd.AddCommand(start.GraflokiStartCmd.MustGetCobraCommand())
+	GraflokiCmd.AddCommand(stop.GraflokiStopCmd.MustGetCobraCommand())
+}

cli/cli/commands/grafloki/start/start.go

@@ -0,0 +1,124 @@
+package start
+
+import (
+	"context"
+	"fmt"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel/args"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel/flags"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/command_str_consts"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/defaults"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/engine_manager"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/grafloki"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/kurtosis_config_getter"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/kurtosis_config/resolved_config"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/out"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_aggregator"
+	"github.com/kurtosis-tech/stacktrace"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	defaultEngineVersion                   = ""
+	restartEngineOnSameVersionIfAnyRunning = true
+)
+
+var GraflokiStartCmd = &lowlevel.LowlevelKurtosisCommand{
+	CommandStr:               command_str_consts.GraflokiStartCmdStr,
+	ShortDescription:         "Starts a Grafana/Loki instance.",
+	LongDescription:          "Starts a Grafana/Loki instance and configures Kurtosis engine to send logs to it.",
+	RunFunc:                  run,
+	Flags:                    nil,
+	Args:                     nil,
+	PostValidationAndRunFunc: nil,
+	PreValidationAndRunFunc:  nil,
+}
+
+func run(
+	ctx context.Context,
+	_ *flags.ParsedFlags,
+	_ *args.ParsedArgs,
+
+) error {
+	clusterConfig, err := kurtosis_config_getter.GetKurtosisClusterConfig()
+	if err != nil {
+		return stacktrace.Propagate(err, "An error occurred getting Kurtosis cluster config.")
+	}
+
+	// NOTE(tedi  04/03/25): If you're wondering why the grafana / loki instance is being started by the CLI (and not in container-engine-lib via KurtosisBackend as with LogsCollector and LogsAggregator), here's why:
+	// 1. now that Kurtosis is purely OSS, it's important to reduce maintenance surface / complexity inside Kurtosis core (Engine, APIContainer, KurtosisBackend, Starlark Engine)
+	// 	  wherever possible infra not essential to Kurtosis core should be built outside of it or at the edges (e.g. client)
+	// 2. the export logs feature was built in service of leveraging existing logging solutions/not rebuilding logging in Kurtosis
+	// 3. having grafloki started/managed by the CLI lets us build on the export logs feature
+	// In other words
+	// putting it in the CLI is saying - “You could set up Grafana and Loki yourself, and then restart the engine to point to it, Kurtosis CLI will do that for you to save you a step”
+	// putting it in Kurtosis core is saying - “Grafana and Loki are core a necessary part of the Kurtosis platform and supports the Kurtosis abstraction/value prop" - which is not the case
+	// https://drawpaintacademy.com/the-bull/
+	var lokiHost string
+	var grafanaUrl string
+	switch clusterConfig.GetClusterType() {
+	case resolved_config.KurtosisClusterType_Docker:
+		lokiHost, grafanaUrl, err = grafloki.StartGrafLokiInDocker(ctx)
+		if err != nil {
+			return stacktrace.Propagate(err, "An error occurred starting Grafana and Loki in Docker.")
+		}
+	case resolved_config.KurtosisClusterType_Kubernetes:
+		lokiHost, grafanaUrl, err = grafloki.StartGrafLokiInKubernetes(ctx)
+		if err != nil {
+			return stacktrace.Propagate(err, "An error occurred starting Grafana and Loki in Kubernetes.")
+		}
+	default:
+		return stacktrace.NewError("Unsupported cluster type: %v", clusterConfig.GetClusterType().String())
+	}
+
+	// This matches the exact configurations here: https://vector.dev/docs/reference/configuration/sinks/loki/
+	lokiSink := map[string]map[string]interface{}{
+		"loki": {
+			"type":     "loki",
+			"endpoint": lokiHost,
+			"encoding": map[string]string{
+				"codec": "json",
+			},
+			"labels": map[string]string{
+				"job": "kurtosis",
+			},
+		},
+	}
+
+	logrus.Infof("Configuring engine to send logs to Loki...")
+	err = restartEngineWithLogsSink(ctx, lokiSink)
+	if err != nil {
+		return stacktrace.Propagate(err, "An error occurred restarting engine to be configured to send logs to Loki.")
+	}
+
+	out.PrintOutLn(fmt.Sprintf("Grafana running at %v", grafanaUrl))
+	return nil
+}
+
+func restartEngineWithLogsSink(ctx context.Context, sink logs_aggregator.Sinks) error {
+	engineManager, err := engine_manager.NewEngineManager(ctx)
+	if err != nil {
+		return stacktrace.Propagate(err, "An error occurred creating an engine manager.")
+	}
+	dontRestartAPIContainers := false
+	_, engineClientCloseFunc, restartEngineErr := engineManager.RestartEngineIdempotently(ctx,
+		defaults.DefaultEngineLogLevel,
+		defaultEngineVersion,
+		restartEngineOnSameVersionIfAnyRunning,
+		defaults.DefaultEngineEnclavePoolSize,
+		defaults.DefaultEnableDebugMode,
+		defaults.DefaultGitHubAuthTokenOverride,
+		dontRestartAPIContainers,
+		defaults.DefaultDomain,
+		defaults.DefaultLogRetentionPeriod,
+		sink)
+	if restartEngineErr != nil {
+		return stacktrace.Propagate(restartEngineErr, "An error occurred restarting the Kurtosis engine")
+	}
+	defer func() {
+		if err = engineClientCloseFunc(); err != nil {
+			logrus.Warnf("Error closing the engine client:\n'%v'", err)
+		}
+	}()
+	return nil
+}

cli/cli/commands/grafloki/stop/stop.go

@@ -0,0 +1,54 @@
+package stop
+
+import (
+	"context"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel/args"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/command_framework/lowlevel/flags"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/command_str_consts"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/grafloki"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/helpers/kurtosis_config_getter"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/kurtosis_config/resolved_config"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/out"
+	"github.com/kurtosis-tech/stacktrace"
+)
+
+var GraflokiStopCmd = &lowlevel.LowlevelKurtosisCommand{
+	CommandStr:               command_str_consts.GraflokiStopCmdStr,
+	ShortDescription:         "Stops a grafana/loki instance.",
+	LongDescription:          "Stop a grafana/loki instance if one already exists.",
+	RunFunc:                  run,
+	Flags:                    nil,
+	Args:                     nil,
+	PreValidationAndRunFunc:  nil,
+	PostValidationAndRunFunc: nil,
+}
+
+func run(
+	ctx context.Context,
+	_ *flags.ParsedFlags,
+	_ *args.ParsedArgs,
+) error {
+	clusterConfig, err := kurtosis_config_getter.GetKurtosisClusterConfig()
+	if err != nil {
+		return stacktrace.Propagate(err, "An error occurred getting Kurtosis cluster config.")
+	}
+
+	switch clusterConfig.GetClusterType() {
+	case resolved_config.KurtosisClusterType_Docker:
+		err := grafloki.StopGrafLokiInDocker(ctx)
+		if err != nil {
+			return stacktrace.Propagate(err, "An error occurred stopping Grafana and Loki containers in Docker.")
+		}
+	case resolved_config.KurtosisClusterType_Kubernetes:
+		err := grafloki.StopGrafLokiInKubernetes(ctx)
+		if err != nil {
+			return stacktrace.Propagate(err, "An error occurred stopping Grafana and Loki containers in Kubernetes.")
+		}
+	default:
+		return stacktrace.NewError("Unsupported cluster type: %v", clusterConfig.GetClusterType().String())
+	}
+
+	out.PrintOutLn("Successfully stopped Grafana and Loki containers.")
+	return nil
+}

cli/cli/commands/kurtosis_context/set/set.go

@@ -148,7 +148,7 @@ func SetContext(
 	}
 
 	dontRestartAPIContainers := false
-	_, engineClientCloseFunc, startEngineErr := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, logrus.InfoLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain, defaults.DefaultLogRetentionPeriod)
+	_, engineClientCloseFunc, startEngineErr := engineManager.StartEngineIdempotentlyWithDefaultVersion(ctx, logrus.InfoLevel, defaults.DefaultEngineEnclavePoolSize, defaults.DefaultGitHubAuthTokenOverride, dontRestartAPIContainers, defaults.DefaultDomain, defaults.DefaultLogRetentionPeriod, defaults.DefaultLogsSinks)
 	if startEngineErr != nil {
 		logrus.Warnf("The context was successfully set to '%s' but Kurtosis failed to start an engine in "+
 			"this new context. A new engine should be started manually with '%s %s %s'. The error was:\n%v",

cli/cli/commands/root.go

@@ -8,6 +8,7 @@ package commands
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/kurtosis-tech/kurtosis/cli/cli/commands/grafloki"
 	"io"
 	"net/http"
 	"os"
@@ -141,6 +142,7 @@ func init() {
 	RootCmd.AddCommand(web.WebCmd.MustGetCobraCommand())
 	RootCmd.AddCommand(_package.PackageCmd)
 	RootCmd.AddCommand(github.GitHubCmd)
+	RootCmd.AddCommand(grafloki.GraflokiCmd)
 }
 
 // ====================================================================================================

cli/cli/defaults/defaults.go

@@ -6,6 +6,7 @@
 package defaults
 
 import (
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_aggregator"
 	"github.com/sirupsen/logrus"
 )
 
@@ -30,5 +31,6 @@ const (
 	DefaultLogRetentionPeriod = "168h"
 )
 
+var DefaultLogsSinks = logs_aggregator.Sinks{}
 var DefaultApiContainerLogLevel = logrus.DebugLevel
 var DefaultEngineLogLevel = logrus.DebugLevel

cli/cli/go.mod

@@ -51,6 +51,8 @@ require (
 	github.com/cli/go-gh/v2 v2.4.1-0.20231120145612-d32c104a9a25
 	github.com/cli/oauth v1.0.1
 	github.com/compose-spec/compose-go v1.17.0
+	github.com/docker/docker v24.0.9+incompatible
+	github.com/docker/go-connections v0.4.0
 	github.com/fatih/color v1.13.0
 	github.com/go-git/go-git/v5 v5.14.0
 	github.com/google/go-github/v50 v50.2.0
@@ -95,8 +97,6 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
 	github.com/distribution/distribution/v3 v3.0.0-20230214150026-36d8c594d7aa // indirect
-	github.com/docker/docker v24.0.9+incompatible // indirect
-	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect