diff --git a/examples/nodeconfigs/custom_cni_subnet.json b/examples/nodeconfigs/custom_cni_subnet.json new file mode 100644 index 00000000..8ec20411 --- /dev/null +++ b/examples/nodeconfigs/custom_cni_subnet.json @@ -0,0 +1,17 @@ +{ + "default_resource_dir":"/tmp/wd", + "machine_pool_size": 1, + "internal_node_host":"10.10.10.1", + "cni": { + "network_name": "fcnet", + "interface_name": "veth0", + "subnet": "10.10.10.0/24" + }, + "machine_template": { + "vcpu_count": 1, + "memsize_mib": 256 + }, + "tags": { + "simple": "true" + } +} diff --git a/internal/models/config.go b/internal/models/config.go index 44b234c5..cc21bf5f 100644 --- a/internal/models/config.go +++ b/internal/models/config.go @@ -2,6 +2,7 @@ package models import ( "errors" + "net/netip" "os" "path/filepath" @@ -9,12 +10,15 @@ import ( agentapi "github.com/synadia-io/nex/internal/agent-api" ) -const DefaultCNINetworkName = "fcnet" -const DefaultCNIInterfaceName = "veth0" -const DefaultInternalNodeHost = "192.168.127.1" -const DefaultInternalNodePort = 9222 -const DefaultNodeMemSizeMib = 256 -const DefaultNodeVcpuCount = 1 +const ( + DefaultCNINetworkName = "fcnet" + DefaultCNIInterfaceName = "veth0" + DefaultCNISubnet = "192.168.127.0/24" + DefaultInternalNodeHost = "192.168.127.1" + DefaultInternalNodePort = 9222 + DefaultNodeMemSizeMib = 256 + DefaultNodeVcpuCount = 1 +) var ( // docker/OCI needs to be explicitly enabled in node configuration @@ -68,6 +72,20 @@ func (c *NodeConfiguration) Validate() bool { c.Errors = append(c.Errors, err) } + cniSubnet, err := netip.ParsePrefix(*c.CNI.Subnet) + if err != nil { + c.Errors = append(c.Errors, err) + } + internalNodeHost, err := netip.ParseAddr(*c.InternalNodeHost) + if err != nil { + c.Errors = append(c.Errors, err) + } + + hostInSubnet := cniSubnet.Contains(internalNodeHost) + if !hostInSubnet { + c.Errors = append(c.Errors, errors.New("internal node host must be in the CNI subnet")) + } + return len(c.Errors) == 0 } @@ -89,6 +107,7 @@ func DefaultNodeConfiguration() NodeConfiguration { BinPath: DefaultCNIBinPath, NetworkName: agentapi.StringOrNil(DefaultCNINetworkName), InterfaceName: agentapi.StringOrNil(DefaultCNIInterfaceName), + Subnet: agentapi.StringOrNil(DefaultCNISubnet), }, // CAUTION: This needs to be the IP of the node server's internal NATS --as visible to the inside of the firecracker VM--. This is not necessarily the address // on which the internal NATS server is actually listening on inside the node. diff --git a/internal/models/node.go b/internal/models/node.go index aa0642bc..93fc868c 100644 --- a/internal/models/node.go +++ b/internal/models/node.go @@ -12,6 +12,7 @@ type CNIDefinition struct { BinPath []string `json:"bin_path"` InterfaceName *string `json:"interface_name"` NetworkName *string `json:"network_name"` + Subnet *string `json:"subnet"` } // Defines the CPU and memory usage of a machine to be configured when it is added to the pool diff --git a/internal/node/prereq.go b/internal/node/prereq.go index 30afe94b..9c3ccce2 100644 --- a/internal/node/prereq.go +++ b/internal/node/prereq.go @@ -3,6 +3,7 @@ package nexnode import ( "archive/tar" "bufio" + "bytes" "compress/gzip" "encoding/json" "fmt" @@ -12,9 +13,11 @@ import ( "path/filepath" "runtime" "strings" + "text/template" "github.com/fatih/color" "github.com/synadia-io/nex/internal/models" + "github.com/synadia-io/nex/internal/node/templates" _ "embed" ) @@ -277,7 +280,6 @@ func CheckPrerequisites(config *models.NodeConfiguration, readonly bool) error { return nil } -// func writeCniConf(fileName string, networkName string) error { func writeCniConf(r *requirement, c *models.NodeConfiguration) error { for _, tF := range r.files { f, err := os.Create(filepath.Join(r.directories[0], tF.name)) @@ -286,19 +288,17 @@ func writeCniConf(r *requirement, c *models.NodeConfiguration) error { } defer f.Close() - var fcConfig map[string]interface{} - err = json.Unmarshal(defaultFcNetConf, &fcConfig) + tmpl, err := template.New("fcnet_conf").Parse(templates.FcnetConfig) if err != nil { return err } - - fcConfig["name"] = c.CNI.NetworkName - raw, err := json.Marshal(fcConfig) + var buffer bytes.Buffer + err = tmpl.Execute(&buffer, c.CNI) if err != nil { return err } - _, err = f.Write(raw) + _, err = f.Write(buffer.Bytes()) if err != nil { return nil } @@ -514,6 +514,3 @@ func decompressTarFromURL(url string, _ string) (*tar.Reader, error) { rawData := tar.NewReader(uncompressedTar) return rawData, nil } - -//go:embed templates/fcnet.conflist -var defaultFcNetConf []byte diff --git a/internal/node/processmanager/spawn_procman.go b/internal/node/processmanager/spawn_procman.go index 2874ce03..2709ac16 100644 --- a/internal/node/processmanager/spawn_procman.go +++ b/internal/node/processmanager/spawn_procman.go @@ -219,8 +219,8 @@ func (s *SpawningProcessManager) spawn() (*spawnedProcess, error) { fmt.Sprintf("NEX_NODE_NATS_PORT=%d", *s.config.InternalNodePort), ) - cmd.Stderr = &procLogEmitter{workloadID: workloadID, log: s.log, stderr: true} - cmd.Stdout = &procLogEmitter{workloadID: workloadID, log: s.log, stderr: false} + cmd.Stderr = &procLogEmitter{workloadID: workloadID, log: s.log.WithGroup(workloadID), stderr: true} + cmd.Stdout = &procLogEmitter{workloadID: workloadID, log: s.log.WithGroup(workloadID), stderr: false} newProc := &spawnedProcess{ ID: workloadID, diff --git a/internal/node/templates/fcnet.conflist b/internal/node/templates/fcnet.conflist index 5360e36a..b349a0dd 100644 --- a/internal/node/templates/fcnet.conflist +++ b/internal/node/templates/fcnet.conflist @@ -15,4 +15,4 @@ "type": "tc-redirect-tap" } ] -} \ No newline at end of file +} diff --git a/internal/node/templates/fcnet_config.go b/internal/node/templates/fcnet_config.go new file mode 100644 index 00000000..0781a99a --- /dev/null +++ b/internal/node/templates/fcnet_config.go @@ -0,0 +1,20 @@ +package templates + +var FcnetConfig string = `{ + "name": "{{.NetworkName}}", + "cniVersion": "0.4.0", + "plugins": [ + { + "type": "ptp", + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "{{.Subnet}}", + "resolvConf": "/etc/resolv.conf" + } + }, + { + "type": "tc-redirect-tap" + } + ] +}` diff --git a/internal/node/workload_mgr.go b/internal/node/workload_mgr.go index 81cc6251..10bf8340 100644 --- a/internal/node/workload_mgr.go +++ b/internal/node/workload_mgr.go @@ -32,7 +32,7 @@ const ( LogSubjectPrefix = "$NEX.logs" WorkloadCacheBucketName = "NEXCACHE" - defaultHandshakeTimeoutMillis = 1500 + defaultHandshakeTimeoutMillis = 5000 nexRuntimeNs = "x-nex-runtime-ns" ) diff --git a/nex/nex.go b/nex/nex.go index e1bbc9bf..8c50c637 100644 --- a/nex/nex.go +++ b/nex/nex.go @@ -11,6 +11,7 @@ import ( "github.com/choria-io/fisk" "github.com/fatih/color" shandler "github.com/jordan-rash/slog-handler" + "github.com/nats-io/nats.go" "github.com/synadia-io/nex/internal/models" nextui "github.com/synadia-io/nex/nex/tui" ) @@ -60,7 +61,7 @@ var ( func init() { _ = versionCheck() - ncli.Flag("server", "NATS server urls").Short('s').Envar("NATS_URL").PlaceHolder("URL").StringVar(&Opts.Servers) + ncli.Flag("server", "NATS server urls").Short('s').Envar("NATS_URL").Default(nats.DefaultURL).StringVar(&Opts.Servers) ncli.Flag("user", "Username or Token").Envar("NATS_USER").PlaceHolder("USER").StringVar(&Opts.Username) ncli.Flag("password", "Password").Envar("NATS_PASSWORD").PlaceHolder("PASSWORD").StringVar(&Opts.Password) ncli.Flag("creds", "User credentials file (JWT authentication)").Envar("NATS_CREDS").PlaceHolder("FILE").StringVar(&Opts.Creds) diff --git a/spec/node_test.go b/spec/node_test.go index fa301783..5354238e 100644 --- a/spec/node_test.go +++ b/spec/node_test.go @@ -220,6 +220,7 @@ var _ = Describe("nex node", func() { BeforeEach(func() { nodeConfig = models.DefaultNodeConfiguration() nodeOpts.ConfigFilepath = path.Join(os.TempDir(), fmt.Sprintf("%d-spec-nex-conf.json", _fixtures.seededRand.Int())) + nodeConfig.NoSandbox = !sandbox }) @@ -268,8 +269,12 @@ var _ = Describe("nex node", func() { }) JustBeforeEach(func() { + var err error + _ = nexnode.CmdPreflight(opts, nodeOpts, ctxx, cancel, log) - node, _ = nexnode.NewNode(opts, nodeOpts, ctxx, cancel, log) + node, err = nexnode.NewNode(opts, nodeOpts, ctxx, cancel, log) + Expect(err).To(BeNil()) + go node.Start() nodeID, _ = node.PublicKey()