Makes CNI config more flexible

examples/nodeconfigs/custom_cni_subnet.json

@@ -0,0 +1,17 @@
+{
+    "default_resource_dir":"/tmp/wd",
+    "machine_pool_size": 1,
+    "internal_node_host":"10.10.10.1",
+    "cni": {
+        "network_name": "fcnet",
+        "interface_name": "veth0",
+        "subnet": "10.10.10.0/24"
+    },
+    "machine_template": {
+        "vcpu_count": 1,
+        "memsize_mib": 256
+    },
+    "tags": {
+        "simple": "true"
+    }
+}

internal/models/config.go

@@ -2,19 +2,23 @@ package models
 
 import (
 	"errors"
+	"net/netip"
 	"os"
 	"path/filepath"
 
 	"github.com/splode/fname"
 	agentapi "github.com/synadia-io/nex/internal/agent-api"
 )
 
-const DefaultCNINetworkName = "fcnet"
-const DefaultCNIInterfaceName = "veth0"
-const DefaultInternalNodeHost = "192.168.127.1"
-const DefaultInternalNodePort = 9222
-const DefaultNodeMemSizeMib = 256
-const DefaultNodeVcpuCount = 1
+const (
+	DefaultCNINetworkName   = "fcnet"
+	DefaultCNIInterfaceName = "veth0"
+	DefaultCNISubnet        = "192.168.127.0/24"
+	DefaultInternalNodeHost = "192.168.127.1"
+	DefaultInternalNodePort = 9222
+	DefaultNodeMemSizeMib   = 256
+	DefaultNodeVcpuCount    = 1
+)
 
 var (
 	// docker/OCI needs to be explicitly enabled in node configuration
@@ -68,6 +72,20 @@ func (c *NodeConfiguration) Validate() bool {
 		c.Errors = append(c.Errors, err)
 	}
 
+	cniSubnet, err := netip.ParsePrefix(*c.CNI.Subnet)
+	if err != nil {
+		c.Errors = append(c.Errors, err)
+	}
+	internalNodeHost, err := netip.ParseAddr(*c.InternalNodeHost)
+	if err != nil {
+		c.Errors = append(c.Errors, err)
+	}
+
+	hostInSubnet := cniSubnet.Contains(internalNodeHost)
+	if !hostInSubnet {
+		c.Errors = append(c.Errors, errors.New("internal node host must be in the CNI subnet"))
+	}
+
 	return len(c.Errors) == 0
 }
 
@@ -89,6 +107,7 @@ func DefaultNodeConfiguration() NodeConfiguration {
 			BinPath:       DefaultCNIBinPath,
 			NetworkName:   agentapi.StringOrNil(DefaultCNINetworkName),
 			InterfaceName: agentapi.StringOrNil(DefaultCNIInterfaceName),
+			Subnet:        agentapi.StringOrNil(DefaultCNISubnet),
 		},
 		// CAUTION: This needs to be the IP of the node server's internal NATS --as visible to the inside of the firecracker VM--. This is not necessarily the address
 		// on which the internal NATS server is actually listening on inside the node.

internal/models/node.go

@@ -12,6 +12,7 @@ type CNIDefinition struct {
 	BinPath       []string `json:"bin_path"`
 	InterfaceName *string  `json:"interface_name"`
 	NetworkName   *string  `json:"network_name"`
+	Subnet        *string  `json:"subnet"`
 }
 
 // Defines the CPU and memory usage of a machine to be configured when it is added to the pool

internal/node/prereq.go

@@ -3,6 +3,7 @@ package nexnode
 import (
 	"archive/tar"
 	"bufio"
+	"bytes"
 	"compress/gzip"
 	"encoding/json"
 	"fmt"
@@ -12,9 +13,11 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"text/template"
 
 	"github.com/fatih/color"
 	"github.com/synadia-io/nex/internal/models"
+	"github.com/synadia-io/nex/internal/node/templates"
 
 	_ "embed"
 )
@@ -277,7 +280,6 @@ func CheckPrerequisites(config *models.NodeConfiguration, readonly bool) error {
 	return nil
 }
 
-// func writeCniConf(fileName string, networkName string) error {
 func writeCniConf(r *requirement, c *models.NodeConfiguration) error {
 	for _, tF := range r.files {
 		f, err := os.Create(filepath.Join(r.directories[0], tF.name))
@@ -286,19 +288,17 @@ func writeCniConf(r *requirement, c *models.NodeConfiguration) error {
 		}
 		defer f.Close()
 
-		var fcConfig map[string]interface{}
-		err = json.Unmarshal(defaultFcNetConf, &fcConfig)
+		tmpl, err := template.New("fcnet_conf").Parse(templates.FcnetConfig)
 		if err != nil {
 			return err
 		}
-
-		fcConfig["name"] = c.CNI.NetworkName
-		raw, err := json.Marshal(fcConfig)
+		var buffer bytes.Buffer
+		err = tmpl.Execute(&buffer, c.CNI)
 		if err != nil {
 			return err
 		}
 
-		_, err = f.Write(raw)
+		_, err = f.Write(buffer.Bytes())
 		if err != nil {
 			return nil
 		}
@@ -514,6 +514,3 @@ func decompressTarFromURL(url string, _ string) (*tar.Reader, error) {
 	rawData := tar.NewReader(uncompressedTar)
 	return rawData, nil
 }
-
-//go:embed templates/fcnet.conflist
-var defaultFcNetConf []byte

internal/node/processmanager/spawn_procman.go

@@ -219,8 +219,8 @@ func (s *SpawningProcessManager) spawn() (*spawnedProcess, error) {
 		fmt.Sprintf("NEX_NODE_NATS_PORT=%d", *s.config.InternalNodePort),
 	)
 
-	cmd.Stderr = &procLogEmitter{workloadID: workloadID, log: s.log, stderr: true}
-	cmd.Stdout = &procLogEmitter{workloadID: workloadID, log: s.log, stderr: false}
+	cmd.Stderr = &procLogEmitter{workloadID: workloadID, log: s.log.WithGroup(workloadID), stderr: true}
+	cmd.Stdout = &procLogEmitter{workloadID: workloadID, log: s.log.WithGroup(workloadID), stderr: false}
 
 	newProc := &spawnedProcess{
 		ID:   workloadID,

internal/node/templates/fcnet.conflist

@@ -15,4 +15,4 @@
       "type": "tc-redirect-tap"
     }
   ]
-}
+}

internal/node/templates/fcnet_config.go

@@ -0,0 +1,20 @@
+package templates
+
+var FcnetConfig string = `{
+  "name": "{{.NetworkName}}",
+  "cniVersion": "0.4.0",
+  "plugins": [
+    {
+      "type": "ptp",
+      "ipMasq": true,
+      "ipam": {
+        "type": "host-local",
+        "subnet": "{{.Subnet}}",
+        "resolvConf": "/etc/resolv.conf"
+      }
+    },
+    {
+      "type": "tc-redirect-tap"
+    }
+  ]
+}`

internal/node/workload_mgr.go

@@ -32,7 +32,7 @@ const (
 	LogSubjectPrefix        = "$NEX.logs"
 	WorkloadCacheBucketName = "NEXCACHE"
 
-	defaultHandshakeTimeoutMillis = 1500
+	defaultHandshakeTimeoutMillis = 5000
 
 	nexRuntimeNs = "x-nex-runtime-ns"
 )

nex/nex.go

@@ -11,6 +11,7 @@ import (
 	"github.com/choria-io/fisk"
 	"github.com/fatih/color"
 	shandler "github.com/jordan-rash/slog-handler"
+	"github.com/nats-io/nats.go"
 	"github.com/synadia-io/nex/internal/models"
 	nextui "github.com/synadia-io/nex/nex/tui"
 )
@@ -60,7 +61,7 @@ var (
 func init() {
 	_ = versionCheck()
 
-	ncli.Flag("server", "NATS server urls").Short('s').Envar("NATS_URL").PlaceHolder("URL").StringVar(&Opts.Servers)
+	ncli.Flag("server", "NATS server urls").Short('s').Envar("NATS_URL").Default(nats.DefaultURL).StringVar(&Opts.Servers)
 	ncli.Flag("user", "Username or Token").Envar("NATS_USER").PlaceHolder("USER").StringVar(&Opts.Username)
 	ncli.Flag("password", "Password").Envar("NATS_PASSWORD").PlaceHolder("PASSWORD").StringVar(&Opts.Password)
 	ncli.Flag("creds", "User credentials file (JWT authentication)").Envar("NATS_CREDS").PlaceHolder("FILE").StringVar(&Opts.Creds)

spec/node_test.go

@@ -220,6 +220,7 @@ var _ = Describe("nex node", func() {
 			BeforeEach(func() {
 				nodeConfig = models.DefaultNodeConfiguration()
 				nodeOpts.ConfigFilepath = path.Join(os.TempDir(), fmt.Sprintf("%d-spec-nex-conf.json", _fixtures.seededRand.Int()))
+
 				nodeConfig.NoSandbox = !sandbox
 			})
 
@@ -268,8 +269,12 @@ var _ = Describe("nex node", func() {
 					})
 
 					JustBeforeEach(func() {
+						var err error
+
 						_ = nexnode.CmdPreflight(opts, nodeOpts, ctxx, cancel, log)
-						node, _ = nexnode.NewNode(opts, nodeOpts, ctxx, cancel, log)
+						node, err = nexnode.NewNode(opts, nodeOpts, ctxx, cancel, log)
+						Expect(err).To(BeNil())
+
 						go node.Start()
 
 						nodeID, _ = node.PublicKey()