Merge pull request #22 from makeopensource/dev

Dev

.github/workflows/test.yml

@@ -0,0 +1,60 @@
+name: Go Lint and Test
+
+on:
+  workflow_dispatch:
+
+# todo
+#  push:
+#    tags:
+#      - 'v*'  # match tags that start with v (like v1.0.0)
+#    branches:
+#      - release
+#      - master
+#      - dev
+#    paths:
+#      - 'src/**'
+#      - .github/workflows/test.yml
+
+jobs:
+  lint:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.24'
+          cache: true
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v6
+        with:
+          version: v1.64
+          working-directory: src
+
+#  test: todo
+#    name: test go
+#    runs-on: ubuntu-latest
+#    steps:
+#      - uses: actions/checkout@v4
+#        with:
+#          fetch-depth: 0
+#
+#      - uses: actions/setup-go@v5
+#        with:
+#          go-version: '1.24'
+#          cache: true
+#
+#      # docker setup for running tests
+#      - name: Set up Docker Buildx
+#        uses: docker/setup-buildx-action@v3
+#
+#      - name: Install dependencies
+#        run: go mod download
+#
+#      - name: Run test
+#        run: |
+#          go test ./service/jobs/ -v -run Test500Jobs

Dockerfile

@@ -40,7 +40,9 @@ WORKDIR /app/
 
 COPY --from=builder /app/leviathan .
 
-ENV IS_DOCKER=true
+ENV LEVIATHAN_IS_DOCKER=true
+# default level info when running in docker
+ENV LEVIATHAN_LOG_LEVEL=info
 
 EXPOSE 9221
 

Justfile

@@ -44,12 +44,22 @@ bdrn:
     just dk
     docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock -v appdata:/app/appdata/ {{imageName}}
 
+alias dc := dclean
+dclean:
+    docker rm -f $(docker ps -aq)
+    docker image prune -ay
+
 dkrn:
 	docker compose up --build
 
 post:
     docker compose --profile post up
 
+# update all go deps
+[working-directory: 'src']
+get:
+    go get -v -u all
+
 # lint go files
 [working-directory: 'src']
 lint:
@@ -63,3 +73,13 @@ tidy:
 [working-directory: 'src']
 vet:
     go vet ./...
+
+# go build and run
+[working-directory: 'src']
+gb:
+    go build -o ../bin/leviathan.exe
+
+# go build
+gr:
+    just gb
+    ./bin/leviathan.exe

src/api/api.go

@@ -16,13 +16,7 @@ import (
 func StartGrpcServer() {
 	mux := setupEndpoints()
 
-	log.Info().
-		Str("build_date", common.BuildDate).
-		Str("build_commit", common.CommitInfo).
-		Str("git_branch", common.Branch).
-		Str("go_version", common.GoVersion).
-		Str("build_version", common.Version).
-		Msg("Leviathan initialized successfully")
+	log.Info().Msg("Leviathan initialized successfully")
 
 	srvAddr := fmt.Sprintf(":%s", common.ServerPort.GetStr())
 	log.Info().Msgf("starting server on %s", srvAddr)

src/common/config.go

@@ -4,16 +4,21 @@ import (
 	"errors"
 	"fmt"
 	"github.com/joho/godotenv"
+	"github.com/makeopensource/leviathan/models"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
 	"os"
 	"path/filepath"
+	"strconv"
 )
 
 func InitConfig() {
-	err := godotenv.Load()
-	if err != nil {
-		log.Warn().Err(err).Msg(".env not found. you can safely ignore this warning if you dont have a .env file")
+	_, ok := os.LookupEnv("LEVIATHAN_IS_DOCKER")
+	if !ok {
+		err := godotenv.Load() // load .env file for non docker env
+		if err != nil {
+			log.Warn().Err(err).Msg(".env not found. ignore this warning if you did not intend to load a .env file")
+		}
 	}
 
 	defer func() {
@@ -27,23 +32,46 @@ func InitConfig() {
 	configDir := getConfigDir(baseDir)
 
 	viper.SetConfigName("config")
-	viper.SetConfigType("json")
+	viper.SetConfigType("toml")
 	viper.AddConfigPath(configDir)
 
-	setupDefaultOptions(configDir)
-	loadPostgresOptions()
+	// ignore any error to setup default vals
+	_ = viper.ReadInConfig()
+
+	setIfEnvPresentOrDefault(
+		loglevelKey,
+		"LEVIATHAN_LOG_LEVEL",
+		"debug",
+	)
 
-	submissionFolderPath := getStringEnvOrDefault("TMP_SUBMISSION_FOLDER", fmt.Sprintf("%s/%s", baseDir, "submissions"))
-	viper.SetDefault(submissionDirKey, submissionFolderPath)
+	loadPostgresOptions()
+	setupDefaultOptions(configDir)
 
-	outputFolderPath := getStringEnvOrDefault("LOG_OUTPUT_FOLDER", fmt.Sprintf("%s/%s", baseDir, "output"))
-	viper.SetDefault(outputDirKey, outputFolderPath)
+	submissionFolderPath := setIfEnvPresentOrDefault(
+		submissionDirKey,
+		"TMP_SUBMISSION_DIR",
+		fmt.Sprintf("%s/%s", baseDir, "submissions"),
+	)
+	outputFolderPath := setIfEnvPresentOrDefault(
+		outputDirKey,
+		"SUBMISSION_OUTPUT_DIR",
+		fmt.Sprintf("%s/%s", baseDir, "output"),
+	)
 
 	err = makeDirectories([]string{submissionFolderPath, outputFolderPath})
+	if err != nil {
+		log.Fatal().Err(err).Msg("unable to make required directories")
+	}
 
 	if err := viper.SafeWriteConfig(); err != nil {
 		var configFileAlreadyExistsError viper.ConfigFileAlreadyExistsError
-		if !errors.As(err, &configFileAlreadyExistsError) {
+		if errors.As(err, &configFileAlreadyExistsError) {
+			// merge any new changes
+			err := viper.WriteConfig()
+			if err != nil {
+				log.Fatal().Err(err).Msg("viper could not write to config file")
+			}
+		} else {
 			log.Fatal().Err(err).Msg("viper could not write to config file")
 		}
 	}
@@ -52,46 +80,23 @@ func InitConfig() {
 		log.Fatal().Err(err).Msg("could not read config file")
 	}
 
-	log.Info().Msgf("watching config file at %s", viper.ConfigFileUsed())
-	viper.WatchConfig()
-
-	// maybe create viper instance and return from this function
-	// future setup in case https://github.com/spf13/viper/issues/1855 is accepted
+	log.Info().Msgf("loaded config from %s", viper.ConfigFileUsed())
 }
 
 func loadPostgresOptions() {
-	enablePost := false
-	if getStringEnvOrDefault("POSTGRES_ENABLE", "false") == "true" {
-		enablePost = true
+	setIfEnvPresentOrDefault(postgresHostKey, "POSTGRES_HOST", "localhost")
+	setIfEnvPresentOrDefault(postgresPortKey, "POSTGRES_PORT", "5432")
+	setIfEnvPresentOrDefault(postgresUserKey, "POSTGRES_USER", "postgres")
+	setIfEnvPresentOrDefault(postgresPassKey, "POSTGRES_PASSWORD", "postgres")
+	setIfEnvPresentOrDefault(postgresDBKey, "POSTGRES_DB", "postgres")
+	setIfEnvPresentOrDefault(postgresSslKey, "POSTGRES_SSL", "disable")
+
+	val, isDefault := getBoolEnvOrDefault("POSTGRES_ENABLE", false)
+	if isDefault {
+		viper.SetDefault(enablePostgresKey, val)
+	} else {
+		viper.Set(enablePostgresKey, val)
 	}
-	viper.SetDefault(
-		enablePostgresKey,
-		enablePost,
-	)
-	viper.SetDefault(
-		postgresHostKey,
-		getStringEnvOrDefault("POSTGRES_HOST", "localhost"),
-	)
-	viper.SetDefault(
-		postgresPortKey,
-		getStringEnvOrDefault("POSTGRES_PORT", "5432"),
-	)
-	viper.SetDefault(
-		postgresUserKey,
-		getStringEnvOrDefault("POSTGRES_USER", "postgres"),
-	)
-	viper.SetDefault(
-		postgresPassKey,
-		getStringEnvOrDefault("POSTGRES_PASSWORD", ""),
-	)
-	viper.SetDefault(
-		postgresDBKey,
-		getStringEnvOrDefault("POSTGRES_DB", "postgres"),
-	)
-	viper.SetDefault(
-		postgresSslKey,
-		getStringEnvOrDefault("POSTGRES_SSL", "disable"),
-	)
 }
 
 func getConfigDir(baseDir string) string {
@@ -103,12 +108,42 @@ func getConfigDir(baseDir string) string {
 	return configDir
 }
 
-func getStringEnvOrDefault(key, defaultVal string) string {
+// uses viper.Set if env var was found,
+//
+// else uses' viper.SetDefault and uses defaultValue
+//
+// this allows us to overwrite any new configration changes passed via env vars,
+// but ignore if no env were passed
+func setIfEnvPresentOrDefault(configKey, envKeyName, defaultValue string) string {
+	val, isDefault := getStringEnvOrDefault(envKeyName, defaultValue)
+	if isDefault {
+		viper.SetDefault(configKey, val)
+	} else {
+		// always overwrite with key
+		viper.Set(configKey, val)
+	}
+
+	return val
+}
+
+func getStringEnvOrDefault(key, defaultVal string) (finalVal string, isDefault bool) {
+	value := os.Getenv(key)
+	if value == "" {
+		return defaultVal, true
+	}
+	return value, false
+}
+
+func getBoolEnvOrDefault(key string, defaultVal bool) (finalVal, isDefault bool) {
 	value := os.Getenv(key)
 	if value == "" {
-		return defaultVal
+		return defaultVal, true
+	}
+	parseBool, err := strconv.ParseBool(value)
+	if err != nil {
+		return defaultVal, true
 	}
-	return value
+	return parseBool, false
 }
 
 func setupDefaultOptions(configDir string) {
@@ -118,6 +153,15 @@ func setupDefaultOptions(configDir string) {
 	viper.SetDefault(serverPortKey, "9221")
 	viper.SetDefault(enableLocalDockerKey, true)
 	viper.SetDefault(concurrentJobsKey, 50)
+	viper.SetDefault(ClientSSHKey, map[string]models.MachineOptions{
+		"example": {
+			Enable: false,
+			Name:   "example",
+			Host:   "http://localhost:8080",
+			User:   "test",
+			Port:   22,
+		},
+	})
 }
 
 func getBaseDir() (string, error) {

src/common/config_keys.go

@@ -6,17 +6,19 @@ import (
 )
 
 const (
-	concurrentJobsKey = "concurrent_jobs"
+	concurrentJobsKey = "jobs.concurrent_jobs"
 
 	apiKeyKey     = "server.apikey"
 	serverPortKey = "server.port"
+	loglevelKey   = "server.log_level"
 
 	// folders
 	logDirKey        = "folder.log_dir"
 	submissionDirKey = "folder.submission_dir"
 	outputDirKey     = "folder.log_output_dir"
 	// docker config
 	enableLocalDockerKey = "clients.enable_local_docker"
+	ClientSSHKey         = "clients.ssh"
 
 	sqliteDbPathKey = "db.sqlite.db_path"
 	// postgres
@@ -31,7 +33,7 @@ const (
 
 var (
 	// internal use
-
+	LogLevel     = Config{loglevelKey}
 	LogDir       = Config{logDirKey}
 	SqliteDbPath = Config{sqliteDbPathKey}