add support for variable reconciliation

README.md

@@ -30,6 +30,7 @@ Flags:
       --delete            Enable delete missing jobs
   -h, --help              help for fs
       --path string       glob pattern relative to the base-dir (default "**/*.nomad")
+      --var-path string   var glob pattern relative to the base-dir (default "**/*.yml")
       --watch             Enable watch mode
 
 Global Flags:
@@ -61,6 +62,7 @@ Flags:
       --ssh-key string                 SSH private key
       --url string                     git repository URL
       --username string                SSH username (default "git")
+      --var-path string                var glob pattern relative to the repository root (default "**/*.yml")
       --watch                          Enable watch mode (default true)
 
 Global Flags:
@@ -153,3 +155,14 @@ EOF
   }
 }
 ```
+
+## Variables
+
+Variables are yml files. All keys and values in items should be of type string.
+
+```yaml
+path: nomad/jobs/jobname
+items:
+  key1: "value1"
+  key2: "value2"
+```

cmd/bootstrap_fs.go

@@ -11,6 +11,7 @@ import (
 type fsFlags struct {
 	base_dir string
 	path     string
+	var_path string
 	watch    bool
 	delete   bool
 }
@@ -21,6 +22,7 @@ func init() {
 	bootstrapCmd.AddCommand(bootstrapFsCmd)
 	bootstrapFsCmd.Flags().StringVar(&fsArgs.base_dir, "base-dir", "./", "Path to the base directory")
 	bootstrapFsCmd.Flags().StringVar(&fsArgs.path, "path", "**/*.nomad", "glob pattern relative to the base-dir")
+	bootstrapFsCmd.Flags().StringVar(&fsArgs.var_path, "var-path", "**/*.yml", "var glob pattern relative to the base-dir")
 	bootstrapFsCmd.Flags().BoolVar(&fsArgs.watch, "watch", false, "Enable watch mode")
 	bootstrapFsCmd.Flags().BoolVar(&fsArgs.delete, "delete", false, "Enable delete missing jobs")
 }
@@ -31,9 +33,10 @@ var bootstrapFsCmd = &cobra.Command{
 	Long:  ``,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return reconcile.Run(reconcile.ReconcileOptions{
-			Path:   fsArgs.path,
-			Watch:  fsArgs.watch,
-			Delete: fsArgs.delete,
+			Path:    fsArgs.path,
+			VarPath: fsArgs.var_path,
+			Watch:   fsArgs.watch,
+			Delete:  fsArgs.delete,
 			Fs: func() (billy.Filesystem, error) {
 				fs := osfs.New(fsArgs.base_dir)
 				return fs, nil

cmd/bootstrap_git.go

@@ -18,6 +18,7 @@ type gitFlags struct {
 	url         string
 	branch      string
 	path        string
+	var_path    string
 	username    string
 	password    string
 	sshkey      string
@@ -33,6 +34,7 @@ func init() {
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.url, "url", "", "git repository URL")
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.branch, "branch", "main", "git branch")
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.path, "path", "**/*.nomad", "glob pattern relative to the repository root")
+	bootstrapGitCmd.Flags().StringVar(&gitArgs.var_path, "var-path", "**/*.yml", "var glob pattern relative to the repository root")
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.username, "username", "git", "SSH username")
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.username, "password", "", "SSH private key password")
 	bootstrapGitCmd.Flags().StringVar(&gitArgs.sshkey, "ssh-key", "", "SSH private key")
@@ -48,9 +50,10 @@ var bootstrapGitCmd = &cobra.Command{
 	// Args:  cobra.MinimumNArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return reconcile.Run(reconcile.ReconcileOptions{
-			Path:   gitArgs.path,
-			Watch:  gitArgs.watch,
-			Delete: gitArgs.delete,
+			Path:    gitArgs.path,
+			VarPath: gitArgs.var_path,
+			Watch:   gitArgs.watch,
+			Delete:  gitArgs.delete,
 			Fs: func() (billy.Filesystem, error) {
 				repositoryURL, err := url.Parse(gitArgs.url)
 				if err != nil {

pkg/nomad/nomad.go

@@ -90,3 +90,46 @@ func (client *Client) DeleteJob(job *nc.Job) error {
 
 	return nil
 }
+
+func (client *Client) GetVariable(name string) (*nc.Variable, error) {
+	variable, _, err := client.nc.Variables().Read(name, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return variable, nil
+}
+
+func (client *Client) UpdateVariable(v *nc.Variable) error {
+	_, _, err := client.nc.Variables().Update(v, nil)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (client *Client) DeleteVariable(path string) error {
+	_, err := client.nc.Variables().Delete(path, nil)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (client *Client) ListVariables() (map[string]*nc.Variable, error) {
+	variablelist, _, err := client.nc.Variables().List(nil)
+	if err != nil {
+		return nil, err
+	}
+
+	variables := make(map[string]*nc.Variable)
+
+	for _, variable := range variablelist {
+		v, _ := client.GetVariable(variable.Path)
+		variables[variable.Path] = v
+	}
+
+	return variables, nil
+}

pkg/reconcile/reconcile.go

@@ -5,17 +5,22 @@ import (
 	"io"
 	"time"
 
+	nc "github.com/hashicorp/nomad/api"
+
 	"nomad-gitops-operator/pkg/nomad"
 
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-billy/v5/util"
+
+	"gopkg.in/yaml.v3"
 )
 
 type ReconcileOptions struct {
-	Path   string
-	Watch  bool
-	Delete bool
-	Fs     func() (billy.Filesystem, error)
+	Path    string
+	VarPath string
+	Watch   bool
+	Delete  bool
+	Fs      func() (billy.Filesystem, error)
 }
 
 func Run(opts ReconcileOptions) error {
@@ -33,15 +38,57 @@ func Run(opts ReconcileOptions) error {
 			return err
 		}
 
-		files, err := util.Glob(fs, opts.Path)
+		varFiles, err := util.Glob(fs, opts.VarPath)
+		if err != nil {
+			return err
+		}
+
+		desiredStateVariables := make(map[string]interface{})
+
+		for _, varPath := range varFiles {
+			f, err := fs.Open(varPath)
+			if err != nil {
+				return err
+			}
+
+			b, err := io.ReadAll(f)
+			if err != nil {
+				return err
+			}
+
+			var variable nc.Variable
+			err = yaml.Unmarshal(b, &variable)
+			if err != nil {
+				return err
+			}
+
+			variable.Items["nomoporater"] = "true"
+
+			_, ok := desiredStateVariables[variable.Path]
+			if ok {
+				fmt.Printf("Skipping duplicate variable [%s][%s]\n", variable.Path, varPath)
+				continue
+			}
+
+			desiredStateVariables[variable.Path] = variable
+
+			// Update variable
+			fmt.Printf("Updating vars [%s]\n", variable.Path)
+			err = client.UpdateVariable(&variable)
+			if err != nil {
+				return err
+			}
+		}
+
+		nomadJobFiles, err := util.Glob(fs, opts.Path)
 		if err != nil {
 			return err
 		}
 
 		desiredStateJobs := make(map[string]interface{})
 
 		// Parse and apply all jobs from within the git repo
-		for _, filePath := range files {
+		for _, filePath := range nomadJobFiles {
 			f, err := fs.Open(filePath)
 			if err != nil {
 				return err
@@ -77,7 +124,7 @@ func Run(opts ReconcileOptions) error {
 			}
 		}
 
-		// List all jobs managed by Monoporator
+		// List all jobs managed by Nomoporator
 		currentStateJobs, err := client.ListJobs()
 		if err != nil {
 			fmt.Printf("Error %s\n", err)
@@ -104,6 +151,32 @@ func Run(opts ReconcileOptions) error {
 			}
 		}
 
+		// List all variables managed by Nomoporator
+		currentStateVariables, err := client.ListVariables()
+		if err != nil {
+			fmt.Printf("Error %s\n", err)
+		}
+
+		// Check if variable has the required metadata
+		// Check if variable is one of the parsed jobs
+		for _, variable := range currentStateVariables {
+
+			if _, isManaged := variable.Items["nomoporater"]; isManaged {
+				// If the variable is managed by Nomoporator and is part of the desired state
+				if _, inDesiredState := desiredStateVariables[variable.Path]; inDesiredState {
+
+				} else {
+					if opts.Delete {
+						fmt.Printf("Deleting variable [%s]\n", variable.Path)
+						err = client.DeleteVariable(variable.Path)
+						if err != nil {
+							fmt.Println(err)
+						}
+					}
+				}
+			}
+		}
+
 		if !opts.Watch {
 			return nil
 		}