Merge pull request #14 from prabirshrestha/bootstrap-fs

add support for bootstrapping local fs path and support for variables

README.md

@@ -15,6 +15,33 @@ NOMAD_CLIENT_CERT - Required with TLS enabled.
 NOMAD_CLIENT_KEY - Required with TLS enabled.
 ```
 
+### Bootstrapping using a local path
+
+> Get help with `./nomoporator bootstrap fs -h`
+
+```
+Bootstrap Nomad using a local path
+
+Usage:
+  nomoperator bootstrap fs [path] [flags]
+
+Flags:
+      --base-dir string   Path to the base directory (default "./")
+      --delete            Enable delete missing jobs
+  -h, --help              help for fs
+      --path string       glob pattern relative to the base-dir (default "**/*.nomad")
+      --var-path string   var glob pattern relative to the base-dir (default "**/*.vars.yml")
+      --watch             Enable watch mode
+
+Global Flags:
+  -a, --address string   Address of the Nomad server
+```
+
+Use it like this:
+```
+./nomoperator bootstrap fs --base-dir /path/to/base/dir --path jobs/*.nomad
+```
+
 ### Bootstrapping using a git repository
 
 > Get help with `./nomoporator bootstrap git -h`
@@ -27,13 +54,16 @@ Usage:
 
 Flags:
       --branch string                  git branch (default "main")
+      --delete                         Enable delete missing jobs (default true)
   -h, --help                           help for git
       --password string                SSH private key password
       --path string                    glob pattern relative to the repository root (default "**/*.nomad")
       --ssh-insecure-ignore-host-key   Ignore insecure SSH host key
       --ssh-key string                 SSH private key
       --url string                     git repository URL
       --username string                SSH username (default "git")
+      --var-path string                var glob pattern relative to the repository root (default "**/*.vars.yml")
+      --watch                          Enable watch mode (default true)
 
 Global Flags:
   -a, --address string   Address of the Nomad server
@@ -125,3 +155,14 @@ EOF
   }
 }
 ```
+
+## Variables
+
+Variables are yml files. All keys and values in items should be of type string.
+
+```yaml
+path: nomad/jobs/jobname
+items:
+  key1: "value1"
+  key2: "value2"
+```

cmd/bootstrap.go

@@ -8,7 +8,7 @@ func init() {
 
 var bootstrapCmd = &cobra.Command{
 	Use:   "bootstrap",
-	Short: "bootstrap a yaml file into a Vault instance",
+	Short: "bootstrap into a Nomad instance",
 	Long:  ``,
 	Args:  cobra.MinimumNArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) error {

cmd/bootstrap_fs.go

@@ -0,0 +1,45 @@
+package cmd
+
+import (
+	"github.com/go-git/go-billy/v5"
+	"github.com/go-git/go-billy/v5/osfs"
+	"github.com/spf13/cobra"
+
+	"nomad-gitops-operator/pkg/reconcile"
+)
+
+type fsFlags struct {
+	base_dir string
+	path     string
+	var_path string
+	watch    bool
+	delete   bool
+}
+
+var fsArgs fsFlags
+
+func init() {
+	bootstrapCmd.AddCommand(bootstrapFsCmd)
+	bootstrapFsCmd.Flags().StringVar(&fsArgs.base_dir, "base-dir", "./", "Path to the base directory")
+	bootstrapFsCmd.Flags().StringVar(&fsArgs.path, "path", "**/*.nomad", "glob pattern relative to the base-dir")
+	bootstrapFsCmd.Flags().StringVar(&fsArgs.var_path, "var-path", "**/*.vars.yml", "var glob pattern relative to the base-dir")
+	bootstrapFsCmd.Flags().BoolVar(&fsArgs.watch, "watch", false, "Enable watch mode")
+	bootstrapFsCmd.Flags().BoolVar(&fsArgs.delete, "delete", false, "Enable delete missing jobs")
+}
+
+var bootstrapFsCmd = &cobra.Command{
+	Use:   "fs [path]",
+	Short: "Bootstrap Nomad using a local path",
+	Long:  ``,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return reconcile.Run(reconcile.ReconcileOptions{
+			Path:    fsArgs.path,
+			VarPath: fsArgs.var_path,
+			Watch:   fsArgs.watch,
+			Delete:  fsArgs.delete,
+			Fs: func() (billy.Filesystem, error) {
+				fs := osfs.New(fsArgs.base_dir)
+				return fs, nil
+			}})
+	},
+}

cmd/bootstrap_git.go

@@ -3,26 +3,28 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"io"
 	"net/url"
 	"time"
 
-	"github.com/go-git/go-billy/v5/util"
+	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-git/v5"
 	"github.com/spf13/cobra"
 
-	"nomad-gitops-operator/pkg/nomad"
+	"nomad-gitops-operator/pkg/reconcile"
 	"nomad-gitops-operator/pkg/repository"
 )
 
 type gitFlags struct {
 	url         string
 	branch      string
 	path        string
+	var_path    string
 	username    string
 	password    string
 	sshkey      string
 	sshinsecure bool
+	watch       bool
+	delete      bool
 }
 
 var gitArgs gitFlags
@@ -32,10 +34,13 @@ func init() {
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.url, "url", "", "git repository URL")
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.branch, "branch", "main", "git branch")
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.path, "path", "**/*.nomad", "glob pattern relative to the repository root")
+	bootstrapGitCmd.Flags().StringVar(&gitArgs.var_path, "var-path", "**/*.vars.yml", "var glob pattern relative to the repository root")
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.username, "username", "git", "SSH username")
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.username, "password", "", "SSH private key password")
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.sshkey, "ssh-key", "", "SSH private key")
 	bootstrapGitCmd.Flags().BoolVar(&gitArgs.sshinsecure, "ssh-insecure-ignore-host-key", false, "Ignore insecure SSH host key")
+	bootstrapGitCmd.Flags().BoolVar(&gitArgs.watch, "watch", true, "Enable watch mode")
+	bootstrapGitCmd.Flags().BoolVar(&gitArgs.watch, "delete", true, "Enable delete missing jobs")
 }
 
 var bootstrapGitCmd = &cobra.Command{
@@ -44,94 +49,29 @@ var bootstrapGitCmd = &cobra.Command{
 	Long:  ``,
 	// Args:  cobra.MinimumNArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) error {
-		// Create Nomad client
-		client, err := nomad.NewClient()
-		if err != nil {
-			fmt.Printf("Error %s\n", err)
-		}
-
-		// Reconcile
-		for true {
-			repositoryURL, err := url.Parse(gitArgs.url)
-			if err != nil {
-				return err
-			}
-
-			ctx, cancel := context.WithTimeout(context.Background(), time.Duration(time.Minute*5))
-			defer cancel()
-
-			worktree, err := repository.CLone(ctx, repositoryURL, gitArgs.branch, gitArgs.username, gitArgs.sshkey, gitArgs.password, gitArgs.sshinsecure)
-
-			if err != nil {
-				fmt.Printf("Error: %s\n", err)
-			}
-			worktree.Pull(&git.PullOptions{RemoteName: "origin"})
-
-			fs := worktree.Filesystem
-			files, err := util.Glob(fs, gitArgs.path)
-			if err != nil {
-				return err
-			}
-
-			desiredStateJobs := make(map[string]interface{})
-
-			// Parse and apply all jobs from within the git repo
-			for _, filePath := range files {
-				f, err := fs.Open(filePath)
+		return reconcile.Run(reconcile.ReconcileOptions{
+			Path:    gitArgs.path,
+			VarPath: gitArgs.var_path,
+			Watch:   gitArgs.watch,
+			Delete:  gitArgs.delete,
+			Fs: func() (billy.Filesystem, error) {
+				repositoryURL, err := url.Parse(gitArgs.url)
 				if err != nil {
-					return err
+					return nil, err
 				}
 
-				b, err := io.ReadAll(f)
-				if err != nil {
-					return err
-				}
+				ctx, cancel := context.WithTimeout(context.Background(), time.Duration(time.Minute*5))
+				defer cancel()
 
-				// Parse job
-				job, err := client.ParseJob(string(b))
+				worktree, err := repository.CLone(ctx, repositoryURL, gitArgs.branch, gitArgs.username, gitArgs.sshkey, gitArgs.password, gitArgs.sshinsecure)
 				if err != nil {
-					// If a parse error occurs we skip the job an continue with the next job
-					fmt.Printf("Failed to parse file [%s]: %s\n", filePath, err)
-					continue
-				}
-				desiredStateJobs[job.GetName()] = job
-
-				// Apply job
-				fmt.Printf("Applying job [%s][%s]\n", job.GetName(), filePath)
-				_, err = client.ApplyJob(job)
-				if err != nil {
-					return err
-				}
-			}
-
-			// List all jobs managed by Monoporator
-			currentStateJobs, err := client.ListJobs()
-			if err != nil {
-				fmt.Printf("Error %s\n", err)
-			}
-
-			// Check if job has the required metadata
-			// Check if job is one of the parsed jobs
-			for _, job := range currentStateJobs {
-				meta := job.GetMeta()
-
-				if _, isManaged := meta["nomoporater"]; isManaged {
-					// If the job is managed by Nomoporator and is part of the desired state
-					if _, inDesiredState := desiredStateJobs[job.GetName()]; inDesiredState {
-
-					} else {
-						fmt.Printf("Deleting job [%s]\n", job.GetName())
-						err = client.DeleteJob(job)
-						if err != nil {
-							fmt.Println(err)
-						}
-					}
+					fmt.Printf("Error: %s\n", err)
 				}
-			}
 
-			time.Sleep(30 * time.Second)
-		}
+				worktree.Pull(&git.PullOptions{RemoteName: "origin"})
 
-		return nil
+				fs := worktree.Filesystem
+				return fs, nil
+			}})
 	},
 }

go.mod

@@ -1,58 +1,63 @@
 module nomad-gitops-operator
 
-go 1.18
+go 1.21
+
+toolchain go1.21.4
 
 require (
-	github.com/go-git/go-billy/v5 v5.4.1
-	github.com/go-git/go-git/v5 v5.8.0
-	github.com/hashicorp/nomad-openapi v0.0.0-20230523183328-33c9d2d9ac36
-	github.com/spf13/cobra v1.7.0
-	github.com/spf13/viper v1.16.0
-	golang.org/x/crypto v0.11.0
+	github.com/go-git/go-billy/v5 v5.5.0
+	github.com/go-git/go-git/v5 v5.10.1
+	github.com/hashicorp/nomad/api v0.0.0-20231208210505-e551814df5f2
+	github.com/spf13/cobra v1.8.0
+	github.com/spf13/viper v1.17.0
+	golang.org/x/crypto v0.16.0
 )
 
 require (
-	cloud.google.com/go v0.110.2 // indirect
-	cloud.google.com/go/iam v1.1.1 // indirect
-	cloud.google.com/go/storage v1.29.0 // indirect
+	dario.cat/mergo v1.0.0 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
-	github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
-	github.com/acomagu/bufpipe v1.0.4 // indirect
-	github.com/cloudflare/circl v1.3.3 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c // indirect
+	github.com/cloudflare/circl v1.3.6 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
-	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/googleapis/gax-go/v2 v2.11.0 // indirect
+	github.com/gorilla/websocket v1.5.1 // indirect
 	github.com/hashicorp/cronexpr v1.1.2 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/hcl v1.0.1-vault-5 // indirect
-	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/go-testing-interface v1.14.2-0.20210821155943-2d9075ca8770 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.9 // indirect
+	github.com/onsi/gomega v1.30.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/sergi/go-diff v1.3.1 // indirect
-	github.com/skeema/knownhosts v1.2.0 // indirect
-	github.com/spf13/afero v1.9.5 // indirect
-	github.com/spf13/cast v1.5.1 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/skeema/knownhosts v1.2.1 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/subosito/gotenv v1.4.2 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.12.0 // indirect
-	golang.org/x/oauth2 v0.10.0 // indirect
-	golang.org/x/sys v0.10.0 // indirect
-	golang.org/x/text v0.11.0 // indirect
-	golang.org/x/tools v0.11.0 // indirect
-	google.golang.org/api v0.126.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.16.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect