From 932487de14023eb17ebfbb4abe78475eb2bd9f4b Mon Sep 17 00:00:00 2001
From: Andrea Fasano <afasano@redhat.com>
Date: Fri, 1 Mar 2024 12:42:38 -0500
Subject: [PATCH 01/14] add agent-add-node and import cluster services

---
 .../agent/files/usr/local/bin/add-node.sh     | 47 +++++++++++++++++++
 .../systemd/units/agent-add-node.service      | 21 +++++++++
 .../agent-import-cluster.service.template     | 28 +++++++++++
 3 files changed, 96 insertions(+)
 create mode 100644 data/data/agent/files/usr/local/bin/add-node.sh
 create mode 100644 data/data/agent/systemd/units/agent-add-node.service
 create mode 100644 data/data/agent/systemd/units/agent-import-cluster.service.template

diff --git a/data/data/agent/files/usr/local/bin/add-node.sh b/data/data/agent/files/usr/local/bin/add-node.sh
new file mode 100644
index 0000000000..49a3e324a8
--- /dev/null
+++ b/data/data/agent/files/usr/local/bin/add-node.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+set -e
+
+# shellcheck disable=SC1091
+source issue_status.sh
+
+BASE_URL="${SERVICE_BASE_URL}api/assisted-install/v2"
+
+cluster_id=""
+while [[ "${cluster_id}" = "" ]]
+do
+    # Get cluster id
+    cluster_id=$(curl -s -S "${BASE_URL}/clusters" | jq -r .[].id)
+    if [[ "${cluster_id}" = "" ]]; then
+        sleep 2
+    fi
+done
+
+printf '\nInfra env id is %s\n' "${INFRA_ENV_ID}" 1>&2
+
+status_issue="90_add-node"
+
+# For some reason the initial role patching doesn't seem to work properly
+echo "Patching host..."
+export HOST_ID=$(curl -s ${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts | jq -r '.[].id')
+curl -X PATCH -d '{"host_role":"worker"}' -H "Content-Type: application/json" ${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID}
+
+# Wait for the current host to be ready
+host_ready=false
+while [[ host_ready == false ]]
+do
+    host_status=$(curl -s -S "${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID}" | jq -r .[].status)
+    if [[ "${host_status}" != "known" ]]; then
+        printf '\\e{yellow}Waiting for the host to be ready' | set_issue "${status_issue}"
+        sleep 10
+    else
+        host_ready=true
+    fi
+done
+
+clear_issue "${status_issue}"
+
+sleep 1m
+
+# Add the current host to the cluster
+curl -X POST -s -S ${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID}/actions/install
+echo "Host installation started" 1>&2
diff --git a/data/data/agent/systemd/units/agent-add-node.service b/data/data/agent/systemd/units/agent-add-node.service
new file mode 100644
index 0000000000..c84940b4f8
--- /dev/null
+++ b/data/data/agent/systemd/units/agent-add-node.service
@@ -0,0 +1,21 @@
+[Unit]
+Description=Adds the current node to an already existing cluster
+Wants=network-online.target
+Requires=apply-host-config.service
+PartOf=assisted-service-pod.service
+After=network-online.target apply-host-config.service
+ConditionPathExists=/etc/assisted/node0
+
+[Service]
+EnvironmentFile=/usr/local/share/assisted-service/assisted-service.env
+EnvironmentFile=/usr/local/share/start-cluster/start-cluster.env
+EnvironmentFile=/etc/assisted/rendezvous-host.env
+ExecStartPre=/usr/local/bin/wait-for-assisted-service.sh
+ExecStart=/usr/local/bin/add-node.sh
+
+KillMode=none
+Type=oneshot
+RemainAfterExit=true
+
+[Install]
+WantedBy=multi-user.target
diff --git a/data/data/agent/systemd/units/agent-import-cluster.service.template b/data/data/agent/systemd/units/agent-import-cluster.service.template
new file mode 100644
index 0000000000..634c560369
--- /dev/null
+++ b/data/data/agent/systemd/units/agent-import-cluster.service.template
@@ -0,0 +1,28 @@
+[Unit]
+Description=Imports an already existing cluster
+Wants=network-online.target assisted-service.service
+PartOf=assisted-service-pod.service
+After=network-online.target assisted-service.service
+ConditionPathExists=/etc/assisted/node0
+
+[Service]
+Environment=PODMAN_SYSTEMD_UNIT=%n
+Environment=OPENSHIFT_INSTALL_RELEASE_IMAGE_MIRROR={{.ReleaseImageMirror}}
+EnvironmentFile=/etc/assisted/rendezvous-host.env
+EnvironmentFile=/usr/local/share/assisted-service/agent-images.env
+EnvironmentFile=/usr/local/share/assisted-service/assisted-service.env
+EnvironmentFile=/etc/assisted/add-nodes.env
+ExecStartPre=/bin/rm -f %t/%n.ctr-id
+ExecStartPre=/usr/local/bin/wait-for-assisted-service.sh
+ExecStart=podman run --net host --cidfile=%t/%n.ctr-id --cgroups=no-conmon --log-driver=journald --rm --pod-id-file=%t/assisted-service-pod.pod-id --replace --name=agent-import-cluster -v /etc/assisted/manifests:/manifests -v /etc/assisted/extra-manifests:/extra-manifests -v /etc/pki/ca-trust:/etc/pki/ca-trust:z {{ if .HaveMirrorConfig }}-v /etc/containers:/etc/containers{{ end }} --env SERVICE_BASE_URL --env OPENSHIFT_INSTALL_RELEASE_IMAGE_MIRROR --env CLUSTER_ID --env CLUSTER_NAME --env CLUSTER_API_VIP_DNS_NAME $SERVICE_IMAGE /usr/local/bin/agent-installer-client importCluster
+ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id
+ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id
+
+KillMode=none
+Type=oneshot
+Restart=on-failure
+RestartSec=30
+RemainAfterExit=true
+
+[Install]
+WantedBy=multi-user.target

From 9716c1f6046e225647e67af616a211f47b995e90 Mon Sep 17 00:00:00 2001
From: Andrea Fasano <afasano@redhat.com>
Date: Fri, 1 Mar 2024 12:56:03 -0500
Subject: [PATCH 02/14] modify Ignition asset to enable add-node workflow
 services

---
 .../agent-register-infraenv.service.template  |  2 +-
 pkg/asset/agent/image/ignition.go             | 52 ++++++++++++++-----
 pkg/asset/agent/image/ignition_test.go        |  3 +-
 pkg/asset/agent/image/releaseimage.go         | 29 +++++++----
 4 files changed, 63 insertions(+), 23 deletions(-)

diff --git a/data/data/agent/systemd/units/agent-register-infraenv.service.template b/data/data/agent/systemd/units/agent-register-infraenv.service.template
index 63f16b9e67..4722f0597f 100644
--- a/data/data/agent/systemd/units/agent-register-infraenv.service.template
+++ b/data/data/agent/systemd/units/agent-register-infraenv.service.template
@@ -2,7 +2,7 @@
 Description=Service that registers the infraenv
 Wants=network-online.target assisted-service.service
 PartOf=assisted-service-pod.service
-After=network-online.target assisted-service.service agent-register-cluster.service
+After=network-online.target assisted-service.service {{ if eq .WorkflowType "install" }}agent-register-cluster.service{{ end }}{{ if eq .WorkflowType "addnodes" }}agent-import-cluster.service{{ end }}
 ConditionPathExists=/etc/assisted/node0
 
 [Service]
diff --git a/pkg/asset/agent/image/ignition.go b/pkg/asset/agent/image/ignition.go
index 129e7685b6..f83a63fc7e 100644
--- a/pkg/asset/agent/image/ignition.go
+++ b/pkg/asset/agent/image/ignition.go
@@ -38,6 +38,7 @@ import (
 	"github.com/openshift/installer/pkg/version"
 )
 
+const addNodesEnvPath = "/etc/assisted/add-nodes.env"
 const rendezvousHostEnvPath = "/etc/assisted/rendezvous-host.env"
 const manifestPath = "/etc/assisted/manifests"
 const hostnamesPath = "/etc/assisted/hostnames"
@@ -73,6 +74,7 @@ type agentTemplateData struct {
 	ImageTypeISO              string
 	PublicKeyPEM              string
 	PrivateKeyPEM             string
+	WorkflowType              workflow.AgentWorkflowType
 }
 
 // Name returns the human-friendly name of the asset.
@@ -140,6 +142,8 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 	imageTypeISO := "full-iso"
 	numMasters := 0
 	numWorkers := 0
+	enabledServices := getDefaultEnabledServices()
+	openshiftVersion := ""
 
 	switch agentWorkflow.Workflow {
 	case workflow.AgentWorkflowTypeInstall:
@@ -158,6 +162,13 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 		// Fetch the required number of master and worker nodes.
 		numMasters = agentManifests.AgentClusterInstall.Spec.ProvisionRequirements.ControlPlaneAgents
 		numWorkers = agentManifests.AgentClusterInstall.Spec.ProvisionRequirements.WorkerAgents
+		// Service
+		enabledServices = append(enabledServices, "agent-register-cluster.service", "start-cluster-installation.service")
+		//Version
+		openshiftVersion, err = version.Version()
+		if err != nil {
+			return err
+		}
 
 	case workflow.AgentWorkflowTypeAddNodes:
 		// In the add-nodes workflow, every node will act independently from the others.
@@ -167,6 +178,13 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 		// Fetch the required number of master and worker nodes.
 		numMasters = 0
 		numWorkers = len(addNodesConfig.Config.Hosts)
+		// Service
+		enabledServices = append(enabledServices, "agent-import-cluster.service", "agent-add-node.service")
+		// Generate add-nodes.env file
+		addNodesEnvFile := ignition.FileFromString(addNodesEnvPath, "root", 0644, getAddNodesEnv(*clusterInfo))
+		config.Storage.Files = append(config.Storage.Files, addNodesEnvFile)
+		// Version
+		openshiftVersion = clusterInfo.Version
 
 	default:
 		return fmt.Errorf("AgentWorkflowType value not supported: %s", agentWorkflow.Workflow)
@@ -190,7 +208,7 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 	if releaseArch == "multi" {
 		releaseArchs = []string{arch.RpmArch(types.ArchitectureARM64), arch.RpmArch(types.ArchitectureAMD64), arch.RpmArch(types.ArchitecturePPC64LE), arch.RpmArch(types.ArchitectureS390X)}
 	}
-	releaseImageList, err := releaseImageList(agentManifests.ClusterImageSet.Spec.ReleaseImage, releaseArch, releaseArchs)
+	releaseImageList, err := releaseImageListWithVersion(agentManifests.ClusterImageSet.Spec.ReleaseImage, releaseArch, releaseArchs, openshiftVersion)
 	if err != nil {
 		return err
 	}
@@ -206,7 +224,7 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 	infraEnvID := uuid.New().String()
 	logrus.Debug("Generated random infra-env id ", infraEnvID)
 
-	osImage, err := getOSImagesInfo(archName)
+	osImage, err := getOSImagesInfoWithVersion(archName, openshiftVersion)
 	if err != nil {
 		return err
 	}
@@ -227,7 +245,7 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 		imageTypeISO,
 		keyPairAsset.PrivateKey,
 		keyPairAsset.PublicKey,
-	)
+		agentWorkflow.Workflow)
 
 	err = bootstrap.AddStorageFiles(&config, "/", "agent/files", agentTemplateData)
 	if err != nil {
@@ -265,7 +283,6 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 		return err
 	}
 
-	enabledServices := getDefaultEnabledServices()
 	// Enable pre-network-manager-config.service only when there are network configs defined
 	if len(agentManifests.StaticNetworkConfigs) != 0 {
 		enabledServices = append(enabledServices, "pre-network-manager-config.service")
@@ -299,7 +316,6 @@ func getDefaultEnabledServices() []string {
 		"agent-interactive-console.service",
 		"agent-interactive-console-serial@.service",
 		"agent-register-infraenv.service",
-		"agent-register-cluster.service",
 		"agent.service",
 		"assisted-service-db.service",
 		"assisted-service-pod.service",
@@ -309,7 +325,6 @@ func getDefaultEnabledServices() []string {
 		"selinux.service",
 		"install-status.service",
 		"set-hostname.service",
-		"start-cluster-installation.service",
 	}
 }
 
@@ -343,7 +358,9 @@ func getTemplateData(name, pullSecret, releaseImageList, releaseImage,
 	infraEnvID string,
 	osImage *models.OsImage,
 	proxy *v1beta1.Proxy,
-	imageTypeISO, privateKey, publicKey string) *agentTemplateData {
+	imageTypeISO,
+	privateKey, publicKey string,
+	workflow workflow.AgentWorkflowType) *agentTemplateData {
 	return &agentTemplateData{
 		ServiceProtocol:           "http",
 		PullSecret:                pullSecret,
@@ -361,6 +378,7 @@ func getTemplateData(name, pullSecret, releaseImageList, releaseImage,
 		ImageTypeISO:              imageTypeISO,
 		PrivateKeyPEM:             privateKey,
 		PublicKeyPEM:              publicKey,
+		WorkflowType:              workflow,
 	}
 }
 
@@ -382,6 +400,13 @@ IMAGE_SERVICE_BASE_URL=%s
 `, nodeZeroIP, serviceBaseURL.String(), imageServiceBaseURL.String())
 }
 
+func getAddNodesEnv(clusterInfo joiner.ClusterInfo) string {
+	return fmt.Sprintf(`CLUSTER_ID=%s
+CLUSTER_NAME=%s
+CLUSTER_API_VIP_DNS_NAME=%s
+`, clusterInfo.ClusterID, clusterInfo.ClusterName, clusterInfo.APIDNSName)
+}
+
 func addStaticNetworkConfig(config *igntypes.Config, staticNetworkConfig []*models.HostStaticNetworkConfig) (err error) {
 	if len(staticNetworkConfig) == 0 {
 		return nil
@@ -526,6 +551,14 @@ func addExtraManifests(config *igntypes.Config, extraManifests *manifests.ExtraM
 }
 
 func getOSImagesInfo(cpuArch string) (*models.OsImage, error) {
+	openshiftVersion, err := version.Version()
+	if err != nil {
+		return nil, err
+	}
+	return getOSImagesInfoWithVersion(cpuArch, openshiftVersion)
+}
+
+func getOSImagesInfoWithVersion(cpuArch string, openshiftVersion string) (*models.OsImage, error) {
 	st, err := rhcos.FetchCoreOSBuild(context.Background())
 	if err != nil {
 		return nil, err
@@ -534,11 +567,6 @@ func getOSImagesInfo(cpuArch string) (*models.OsImage, error) {
 	osImage := &models.OsImage{
 		CPUArchitecture: &cpuArch,
 	}
-
-	openshiftVersion, err := version.Version()
-	if err != nil {
-		return nil, err
-	}
 	osImage.OpenshiftVersion = &openshiftVersion
 
 	streamArch, err := st.GetArchitecture(cpuArch)
diff --git a/pkg/asset/agent/image/ignition_test.go b/pkg/asset/agent/image/ignition_test.go
index 02070bdef3..82afa6a5f2 100644
--- a/pkg/asset/agent/image/ignition_test.go
+++ b/pkg/asset/agent/image/ignition_test.go
@@ -92,7 +92,7 @@ func TestIgnition_getTemplateData(t *testing.T) {
 	privateKey := "-----BEGIN EC PUBLIC KEY-----\nMFkwEwYHKoAiDHV4tg==\n-----END EC PUBLIC KEY-----\n"
 	publicKey := "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIOSCfDNmx0qe6dncV4tg==\n-----END EC PRIVATE KEY-----\n"
 
-	templateData := getTemplateData(clusterName, pullSecret, releaseImageList, releaseImage, releaseImageMirror, haveMirrorConfig, publicContainerRegistries, agentClusterInstall.Spec.ProvisionRequirements.ControlPlaneAgents, agentClusterInstall.Spec.ProvisionRequirements.WorkerAgents, infraEnvID, osImage, proxy, "minimal-iso", privateKey, publicKey)
+	templateData := getTemplateData(clusterName, pullSecret, releaseImageList, releaseImage, releaseImageMirror, haveMirrorConfig, publicContainerRegistries, agentClusterInstall.Spec.ProvisionRequirements.ControlPlaneAgents, agentClusterInstall.Spec.ProvisionRequirements.WorkerAgents, infraEnvID, osImage, proxy, "minimal-iso", privateKey, publicKey, workflow.AgentWorkflowTypeInstall)
 	assert.Equal(t, clusterName, templateData.ClusterName)
 	assert.Equal(t, "http", templateData.ServiceProtocol)
 	assert.Equal(t, pullSecret, templateData.PullSecret)
@@ -390,6 +390,7 @@ func commonFiles() []string {
 		"/usr/local/bin/issue_status.sh",
 		"/usr/local/bin/load-config-iso.sh",
 		"/etc/udev/rules.d/80-agent-config-image.rules",
+		"/usr/local/bin/add-node.sh",
 	}
 }
 
diff --git a/pkg/asset/agent/image/releaseimage.go b/pkg/asset/agent/image/releaseimage.go
index 5e32a3e178..ed9f3aa9f5 100644
--- a/pkg/asset/agent/image/releaseimage.go
+++ b/pkg/asset/agent/image/releaseimage.go
@@ -21,21 +21,16 @@ func isDigest(pullspec string) bool {
 	return regexp.MustCompile(`.*sha256:[a-fA-F0-9]{64}$`).MatchString(pullspec)
 }
 
-func releaseImageFromPullSpec(pullSpec, arch string, archs []string) (releaseImage, error) {
+func releaseImageFromPullSpec(pullSpec string, arch string, archs []string, version string) (releaseImage, error) {
 	// When the pullspec it's a digest let's use the current version
 	// stored in the installer
 	if isDigest(pullSpec) {
-		versionString, err := version.Version()
-		if err != nil {
-			return releaseImage{}, err
-		}
-
 		return releaseImage{
-			ReleaseVersion: versionString,
+			ReleaseVersion: version,
 			Arch:           arch,
 			Archs:          archs,
 			PullSpec:       pullSpec,
-			Tag:            versionString,
+			Tag:            version,
 		}, nil
 	}
 
@@ -62,7 +57,23 @@ func releaseImageFromPullSpec(pullSpec, arch string, archs []string) (releaseIma
 }
 
 func releaseImageList(pullSpec, arch string, archs []string) (string, error) {
-	relImage, err := releaseImageFromPullSpec(pullSpec, arch, archs)
+	versionString, err := version.Version()
+	if err != nil {
+		return "", err
+	}
+
+	relImage, err := releaseImageFromPullSpec(pullSpec, arch, archs, versionString)
+	if err != nil {
+		return "", err
+	}
+
+	imageList := []interface{}{relImage}
+	text, err := json.Marshal(imageList)
+	return string(text), err
+}
+
+func releaseImageListWithVersion(pullSpec, arch string, archs []string, openshiftVersion string) (string, error) {
+	relImage, err := releaseImageFromPullSpec(pullSpec, arch, archs, openshiftVersion)
 	if err != nil {
 		return "", err
 	}

From 3c70dc7c237d009e71a86528d5dce9a81f574ce7 Mon Sep 17 00:00:00 2001
From: Andrea Fasano <afasano@redhat.com>
Date: Wed, 6 Mar 2024 09:42:03 -0500
Subject: [PATCH 03/14] retrieve os image data in ClusterInfo

---
 pkg/asset/agent/joiner/clusterinfo.go      | 46 ++++++++++++++++++++
 pkg/asset/agent/joiner/clusterinfo_test.go | 50 +++++++++++++++++++++-
 2 files changed, 94 insertions(+), 2 deletions(-)

diff --git a/pkg/asset/agent/joiner/clusterinfo.go b/pkg/asset/agent/joiner/clusterinfo.go
index 8298eeacf0..fb9b3ae1fa 100644
--- a/pkg/asset/agent/joiner/clusterinfo.go
+++ b/pkg/asset/agent/joiner/clusterinfo.go
@@ -2,6 +2,7 @@ package joiner
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -11,6 +12,8 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/yaml"
 
+	"github.com/coreos/stream-metadata-go/arch"
+	"github.com/coreos/stream-metadata-go/stream"
 	hiveext "github.com/openshift/assisted-service/api/hiveextension/v1beta1"
 	configclient "github.com/openshift/client-go/config/clientset/versioned"
 	"github.com/openshift/installer/pkg/asset"
@@ -40,6 +43,8 @@ type ClusterInfo struct {
 	DeprecatedImageContentSources []types.ImageContentSource
 	PlatformType                  hiveext.PlatformType
 	SSHKey                        string
+	OSImage                       *stream.Stream
+	OSImageLocation               string
 }
 
 var _ asset.WritableAsset = (*ClusterInfo)(nil)
@@ -97,6 +102,10 @@ func (ci *ClusterInfo) Generate(dependencies asset.Parents) error {
 	if err != nil {
 		return err
 	}
+	err = ci.retrieveOsImage()
+	if err != nil {
+		return err
+	}
 
 	ci.Namespace = "cluster0"
 
@@ -223,6 +232,43 @@ func (ci *ClusterInfo) retrieveInstallConfigData() error {
 	return nil
 }
 
+func (ci *ClusterInfo) retrieveOsImage() error {
+	clusterConfig, err := ci.Client.CoreV1().ConfigMaps("openshift-machine-config-operator").Get(context.Background(), "coreos-bootimages", metav1.GetOptions{})
+	if err != nil {
+		if errors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+	data, ok := clusterConfig.Data["stream"]
+	if !ok {
+		return fmt.Errorf("cannot find stream data")
+	}
+
+	var st stream.Stream
+	if err := json.Unmarshal([]byte(data), &st); err != nil {
+		return fmt.Errorf("failed to parse CoreOS stream metadata: %w", err)
+	}
+	ci.OSImage = &st
+
+	clusterArch := arch.RpmArch(ci.Architecture)
+	streamArch, err := st.GetArchitecture(clusterArch)
+	if err != nil {
+		return err
+	}
+	metal, ok := streamArch.Artifacts["metal"]
+	if !ok {
+		return fmt.Errorf("stream data not found for 'metal' artifact")
+	}
+	format, ok := metal.Formats["iso"]
+	if !ok {
+		return fmt.Errorf("no ISO found to download for %s", clusterArch)
+	}
+	ci.OSImageLocation = format.Disk.Location
+
+	return nil
+}
+
 // Files returns the files generated by the asset.
 func (*ClusterInfo) Files() []*asset.File {
 	return []*asset.File{}
diff --git a/pkg/asset/agent/joiner/clusterinfo_test.go b/pkg/asset/agent/joiner/clusterinfo_test.go
index ce2d4e53cf..2134fbdc11 100644
--- a/pkg/asset/agent/joiner/clusterinfo_test.go
+++ b/pkg/asset/agent/joiner/clusterinfo_test.go
@@ -1,8 +1,10 @@
 package joiner
 
 import (
+	"encoding/json"
 	"testing"
 
+	"github.com/coreos/stream-metadata-go/stream"
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -103,6 +105,15 @@ func TestClusterInfo_Generate(t *testing.T) {
 						"install-config": makeInstallConfig(t),
 					},
 				},
+				&corev1.ConfigMap{
+					ObjectMeta: v1.ObjectMeta{
+						Name:      "coreos-bootimages",
+						Namespace: "openshift-machine-config-operator",
+					},
+					Data: map[string]string{
+						"stream": makeCoreOsBootImages(t, buildStreamData()),
+					},
+				},
 			},
 			expectedClusterInfo: ClusterInfo{
 				ClusterID:    "1b5ba46b-7e56-47b1-a326-a9eebddfb38c",
@@ -127,8 +138,10 @@ func TestClusterInfo_Generate(t *testing.T) {
 						},
 					},
 				},
-				PlatformType: v1beta1.BareMetalPlatformType,
-				SSHKey:       "my-ssh-key",
+				PlatformType:    v1beta1.BareMetalPlatformType,
+				SSHKey:          "my-ssh-key",
+				OSImage:         buildStreamData(),
+				OSImageLocation: "http://my-coreosimage-url/416.94.202402130130-0",
 			},
 		},
 	}
@@ -164,10 +177,43 @@ func TestClusterInfo_Generate(t *testing.T) {
 			assert.Equal(t, tc.expectedClusterInfo.DeprecatedImageContentSources, clusterInfo.DeprecatedImageContentSources)
 			assert.Equal(t, tc.expectedClusterInfo.PlatformType, clusterInfo.PlatformType)
 			assert.Equal(t, tc.expectedClusterInfo.SSHKey, clusterInfo.SSHKey)
+			assert.Equal(t, tc.expectedClusterInfo.OSImageLocation, clusterInfo.OSImageLocation)
+			assert.Equal(t, tc.expectedClusterInfo.OSImage, clusterInfo.OSImage)
 		})
 	}
 }
 
+func buildStreamData() *stream.Stream {
+	return &stream.Stream{
+		Architectures: map[string]stream.Arch{
+			"x86_64": {
+				Artifacts: map[string]stream.PlatformArtifacts{
+					"metal": {
+						Release: "416.94.202402130130-0",
+						Formats: map[string]stream.ImageFormat{
+							"iso": {
+								Disk: &stream.Artifact{
+									Location: "http://my-coreosimage-url/416.94.202402130130-0",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func makeCoreOsBootImages(t *testing.T, st *stream.Stream) string {
+	t.Helper()
+	data, err := json.Marshal(st)
+	if err != nil {
+		t.Error(err)
+	}
+
+	return string(data)
+}
+
 func makeInstallConfig(t *testing.T) string {
 	t.Helper()
 	ic := &types.InstallConfig{

From a890cb1084b50060f542bb3c19695917f4270663 Mon Sep 17 00:00:00 2001
From: Andrea Fasano <afasano@redhat.com>
Date: Wed, 6 Mar 2024 09:42:21 -0500
Subject: [PATCH 04/14] allow Ignition and BaseIso assets to retrieve coreos
 metadata from ClusterInfo

This is required to avoid the node-joiner being tied to a specific openshift version, as it happens for the installer when using pinned metadata
---
 pkg/asset/agent/image/agentartifacts.go       |   2 +-
 pkg/asset/agent/image/baseiso.go              |  97 ++++++----
 pkg/asset/agent/image/baseiso_test.go         | 182 +++++++++++++++++-
 pkg/asset/agent/image/ignition.go             |  22 +--
 pkg/asset/agent/image/releaseextract.go       |  15 +-
 .../agent/image/unconfigured_ignition.go      |   7 +-
 6 files changed, 261 insertions(+), 64 deletions(-)

diff --git a/pkg/asset/agent/image/agentartifacts.go b/pkg/asset/agent/image/agentartifacts.go
index 305dbca0db..89058d2538 100644
--- a/pkg/asset/agent/image/agentartifacts.go
+++ b/pkg/asset/agent/image/agentartifacts.go
@@ -92,7 +92,7 @@ func (a *AgentArtifacts) Generate(dependencies asset.Parents) error {
 func (a *AgentArtifacts) fetchAgentTuiFiles(releaseImage string, pullSecret string, mirrorConfig []mirror.RegistriesConfig) ([]string, error) {
 	release := NewRelease(
 		Config{MaxTries: OcDefaultTries, RetryDelay: OcDefaultRetryDelay},
-		releaseImage, pullSecret, mirrorConfig)
+		releaseImage, pullSecret, mirrorConfig, nil)
 
 	agentTuiFilenames := []string{"/usr/bin/agent-tui", "/usr/lib64/libnmstate.so.*"}
 	files := []string{}
diff --git a/pkg/asset/agent/image/baseiso.go b/pkg/asset/agent/image/baseiso.go
index e3434a0237..d3cf08adc9 100644
--- a/pkg/asset/agent/image/baseiso.go
+++ b/pkg/asset/agent/image/baseiso.go
@@ -15,8 +15,10 @@ import (
 
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/agent"
+	"github.com/openshift/installer/pkg/asset/agent/joiner"
 	"github.com/openshift/installer/pkg/asset/agent/manifests"
 	"github.com/openshift/installer/pkg/asset/agent/mirror"
+	"github.com/openshift/installer/pkg/asset/agent/workflow"
 	"github.com/openshift/installer/pkg/rhcos"
 	"github.com/openshift/installer/pkg/rhcos/cache"
 	"github.com/openshift/installer/pkg/types"
@@ -24,11 +26,18 @@ import (
 
 // BaseIso generates the base ISO file for the image
 type BaseIso struct {
-	File *asset.File
+	File         *asset.File
+	streamGetter CoreOSBuildFetcher
+	ocRelease    Release
 }
 
+type CoreOSBuildFetcher func(ctx context.Context) (*stream.Stream, error)
+
 var (
-	baseIsoFilename = ""
+	baseIsoFilename           = ""
+	DefaultCoreOSStreamGetter = func(ctx context.Context) (*stream.Stream, error) {
+		return rhcos.FetchCoreOSBuild(ctx)
+	}
 )
 
 var _ asset.WritableAsset = (*BaseIso)(nil)
@@ -38,26 +47,12 @@ func (i *BaseIso) Name() string {
 	return "BaseIso Image"
 }
 
-// getIsoFile is a pluggable function that gets the base ISO file
-type getIsoFile func(archName string) (string, error)
-
-type getIso struct {
-	getter getIsoFile
-}
-
-func newGetIso(getter getIsoFile) *getIso {
-	return &getIso{getter: getter}
-}
-
-// GetIsoPluggable defines the method to use get the baseIso file
-var GetIsoPluggable = downloadIso
-
-func getMetalArtifact(archName string) (stream.PlatformArtifacts, error) {
+func (i *BaseIso) getMetalArtifact(archName string) (stream.PlatformArtifacts, error) {
 	ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second)
 	defer cancel()
 
 	// Get the ISO to use from rhcos.json
-	st, err := rhcos.FetchCoreOSBuild(ctx)
+	st, err := i.streamGetter(ctx)
 	if err != nil {
 		return stream.PlatformArtifacts{}, err
 	}
@@ -76,8 +71,8 @@ func getMetalArtifact(archName string) (stream.PlatformArtifacts, error) {
 }
 
 // Download the ISO using the URL in rhcos.json.
-func downloadIso(archName string) (string, error) {
-	metal, err := getMetalArtifact(archName)
+func (i *BaseIso) downloadIso(archName string) (string, error) {
+	metal, err := i.getMetalArtifact(archName)
 	if err != nil {
 		return "", err
 	}
@@ -99,7 +94,7 @@ func downloadIso(archName string) (string, error) {
 
 // Fetch RootFS URL using the rhcos.json.
 func (i *BaseIso) getRootFSURL(archName string) (string, error) {
-	metal, err := getMetalArtifact(archName)
+	metal, err := i.getMetalArtifact(archName)
 	if err != nil {
 		return "", err
 	}
@@ -115,6 +110,8 @@ func (i *BaseIso) getRootFSURL(archName string) (string, error) {
 // Dependencies returns dependencies used by the asset.
 func (i *BaseIso) Dependencies() []asset.Asset {
 	return []asset.Asset{
+		&workflow.AgentWorkflow{},
+		&joiner.ClusterInfo{},
 		&manifests.AgentManifests{},
 		&agent.OptionalInstallConfig{},
 		&mirror.RegistriesConf{},
@@ -132,7 +129,7 @@ func (i *BaseIso) checkReleasePayloadBaseISOVersion(r Release, archName string)
 	}
 
 	// Get pinned version from installer
-	metal, err := getMetalArtifact(archName)
+	metal, err := i.getMetalArtifact(archName)
 	if err != nil {
 		logrus.Warnf("unable to determine base ISO version: %s", err.Error())
 		return
@@ -147,10 +144,6 @@ func (i *BaseIso) checkReleasePayloadBaseISOVersion(r Release, archName string)
 // Generate the baseIso
 func (i *BaseIso) Generate(dependencies asset.Parents) error {
 
-	// TODO - if image registry location is defined in InstallConfig,
-	// ic := &agent.OptionalInstallConfig{}
-	// p.Get(ic)
-
 	var err error
 	var baseIsoFileName string
 
@@ -158,6 +151,7 @@ func (i *BaseIso) Generate(dependencies asset.Parents) error {
 		logrus.Warn("Found override for OS Image. Please be warned, this is not advised")
 		baseIsoFileName, err = cache.DownloadImageFile(urlOverride, cache.AgentApplicationName)
 	} else {
+		i.setStreamGetter(dependencies)
 		baseIsoFileName, err = i.retrieveBaseIso(dependencies)
 	}
 
@@ -171,12 +165,43 @@ func (i *BaseIso) Generate(dependencies asset.Parents) error {
 	return errors.Wrap(err, "failed to get base ISO image")
 }
 
+func (i *BaseIso) setStreamGetter(dependencies asset.Parents) {
+	if i.streamGetter != nil {
+		return
+	}
+
+	agentWorkflow := &workflow.AgentWorkflow{}
+	clusterInfo := &joiner.ClusterInfo{}
+	dependencies.Get(agentWorkflow, clusterInfo)
+
+	i.streamGetter = DefaultCoreOSStreamGetter
+	if agentWorkflow.Workflow == workflow.AgentWorkflowTypeAddNodes {
+		i.streamGetter = func(ctx context.Context) (*stream.Stream, error) {
+			return clusterInfo.OSImage, nil
+		}
+	}
+}
+
+func (i *BaseIso) getRelease(agentManifests *manifests.AgentManifests, registriesConf *mirror.RegistriesConf) Release {
+	if i.ocRelease != nil {
+		return i.ocRelease
+	}
+
+	releaseImage := agentManifests.ClusterImageSet.Spec.ReleaseImage
+	pullSecret := agentManifests.GetPullSecretData()
+
+	i.ocRelease = NewRelease(
+		Config{MaxTries: OcDefaultTries, RetryDelay: OcDefaultRetryDelay},
+		releaseImage, pullSecret, registriesConf.MirrorConfig, i.streamGetter)
+
+	return i.ocRelease
+}
+
 func (i *BaseIso) retrieveBaseIso(dependencies asset.Parents) (string, error) {
 	// use the GetIso function to get the BaseIso from the release payload
 	agentManifests := &manifests.AgentManifests{}
-	dependencies.Get(agentManifests)
-	var baseIsoFileName string
-	var err error
+	registriesConf := &mirror.RegistriesConf{}
+	dependencies.Get(agentManifests, registriesConf)
 
 	// Default iso archName to x86_64.
 	archName := arch.RpmArch(types.ArchitectureAMD64)
@@ -186,18 +211,11 @@ func (i *BaseIso) retrieveBaseIso(dependencies asset.Parents) (string, error) {
 		if agentManifests.InfraEnv.Spec.CpuArchitecture != "" {
 			archName = agentManifests.InfraEnv.Spec.CpuArchitecture
 		}
-		releaseImage := agentManifests.ClusterImageSet.Spec.ReleaseImage
-		pullSecret := agentManifests.GetPullSecretData()
-		registriesConf := &mirror.RegistriesConf{}
-		dependencies.Get(agentManifests, registriesConf)
 
 		// If we have the image registry location and 'oc' command is available then get from release payload
-		ocRelease := NewRelease(
-			Config{MaxTries: OcDefaultTries, RetryDelay: OcDefaultRetryDelay},
-			releaseImage, pullSecret, registriesConf.MirrorConfig)
-
+		ocRelease := i.getRelease(agentManifests, registriesConf)
 		logrus.Info("Extracting base ISO from release payload")
-		baseIsoFileName, err = ocRelease.GetBaseIso(archName)
+		baseIsoFileName, err := ocRelease.GetBaseIso(archName)
 		if err == nil {
 			i.checkReleasePayloadBaseISOVersion(ocRelease, archName)
 
@@ -216,8 +234,7 @@ func (i *BaseIso) retrieveBaseIso(dependencies asset.Parents) (string, error) {
 	}
 
 	logrus.Info("Downloading base ISO")
-	isoGetter := newGetIso(GetIsoPluggable)
-	return isoGetter.getter(archName)
+	return i.downloadIso(archName)
 }
 
 // Files returns the files generated by the asset.
diff --git a/pkg/asset/agent/image/baseiso_test.go b/pkg/asset/agent/image/baseiso_test.go
index 26a4c342b9..1f92a893cd 100644
--- a/pkg/asset/agent/image/baseiso_test.go
+++ b/pkg/asset/agent/image/baseiso_test.go
@@ -1,20 +1,196 @@
 package image
 
 import (
+	"context"
+	"crypto/rand"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path"
 	"testing"
 
+	"github.com/coreos/stream-metadata-go/stream"
 	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
 
+	"github.com/openshift/assisted-service/api/v1beta1"
+	v1 "github.com/openshift/hive/apis/hive/v1"
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/agent"
+	"github.com/openshift/installer/pkg/asset/agent/joiner"
 	"github.com/openshift/installer/pkg/asset/agent/manifests"
+	"github.com/openshift/installer/pkg/asset/agent/mirror"
+	"github.com/openshift/installer/pkg/asset/agent/workflow"
 )
 
-func TestInfraBaseIso_Generate(t *testing.T) {
+func setupEmbeddedResources(t *testing.T) {
+	workingDirectory, err := os.Getwd()
+	assert.NoError(t, err)
+	err = os.Chdir(path.Join(workingDirectory, "../../../../data"))
+	assert.NoError(t, err)
+}
+
+func TestBaseIso_Generate(t *testing.T) {
+	setupEmbeddedResources(t)
+	ocReleaseImage := "416.94.202402130130-0"
+	ocBaseIsoFilename := "openshift-4.16"
+
+	cases := []struct {
+		name                       string
+		dependencies               []asset.Asset
+		envVarOsImageOverrideValue string
+		getIsoError                error
+		expectedBaseIsoFilename    string
+		expectedError              string
+	}{
+		{
+			name:                       "os image override",
+			envVarOsImageOverrideValue: "openshift-4.15",
+			expectedBaseIsoFilename:    "openshift-4.15",
+		},
+		{
+			name: "default",
+			dependencies: []asset.Asset{
+				&workflow.AgentWorkflow{Workflow: workflow.AgentWorkflowTypeInstall},
+				&joiner.ClusterInfo{},
+				&manifests.AgentManifests{
+					InfraEnv: &v1beta1.InfraEnv{},
+					ClusterImageSet: &v1.ClusterImageSet{
+						Spec: v1.ClusterImageSetSpec{
+							ReleaseImage: ocReleaseImage,
+						},
+					},
+					PullSecret: &corev1.Secret{
+						StringData: map[string]string{
+							".dockerconfigjson": "supersecret",
+						},
+					},
+				},
+				&mirror.RegistriesConf{},
+			},
+			expectedBaseIsoFilename: ocBaseIsoFilename,
+		},
+		{
+			name: "direct download if oc is not available",
+			dependencies: []asset.Asset{
+				&workflow.AgentWorkflow{Workflow: workflow.AgentWorkflowTypeInstall},
+				&joiner.ClusterInfo{},
+				&manifests.AgentManifests{
+					InfraEnv: &v1beta1.InfraEnv{},
+					ClusterImageSet: &v1.ClusterImageSet{
+						Spec: v1.ClusterImageSetSpec{
+							ReleaseImage: ocReleaseImage,
+						},
+					},
+					PullSecret: &corev1.Secret{
+						StringData: map[string]string{
+							".dockerconfigjson": "supersecret",
+						},
+					},
+				},
+				&mirror.RegistriesConf{},
+			},
+			getIsoError:             &exec.Error{"", exec.ErrNotFound},
+			expectedBaseIsoFilename: ocReleaseImage,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			dependencies := asset.Parents{}
+			dependencies.Add(tc.dependencies...)
+
+			// Setup a fake http server, to serve the future download request.
+			svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				// Answer with a fixed size randomly filled buffer
+				buffer := make([]byte, 1024)
+				rand.Read(buffer)
+				w.Write(buffer)
+			}))
+			defer svr.Close()
+			// Creates a tmp folder to store the .cache downloaded images.
+			tmpPath, err := os.MkdirTemp("", "agent-baseiso-test")
+			assert.NoError(t, err)
+			previousXdgCacheHomeValue := os.Getenv("XDG_CACHE_HOME")
+			os.Setenv("XDG_CACHE_HOME", tmpPath)
+			// Set the image override if defined
+			previousOpenshiftInstallOsImageOverrideValue := os.Getenv("OPENSHIFT_INSTALL_OS_IMAGE_OVERRIDE")
+			if tc.envVarOsImageOverrideValue != "" {
+				newOsImageOverride := fmt.Sprintf("%s/%s", svr.URL, tc.envVarOsImageOverrideValue)
+				os.Setenv("OPENSHIFT_INSTALL_OS_IMAGE_OVERRIDE", newOsImageOverride)
+			}
+			// Cleanup on exit.
+			defer func() {
+				err := os.Setenv("XDG_CACHE_HOME", previousXdgCacheHomeValue)
+				assert.NoError(t, err)
+				err = os.Setenv("OPENSHIFT_INSTALL_OS_IMAGE_OVERRIDE", previousOpenshiftInstallOsImageOverrideValue)
+				assert.NoError(t, err)
+				err = os.RemoveAll(tmpPath)
+				assert.NoError(t, err)
+			}()
+
+			baseIso := &BaseIso{
+				ocRelease: &mockRelease{
+					isoBaseVersion:  ocReleaseImage,
+					baseIsoFileName: ocBaseIsoFilename,
+					baseIsoError:    tc.getIsoError,
+				},
+				streamGetter: func(ctx context.Context) (*stream.Stream, error) {
+					return &stream.Stream{
+						Architectures: map[string]stream.Arch{
+							"x86_64": {
+								Artifacts: map[string]stream.PlatformArtifacts{
+									"metal": {
+										Release: ocReleaseImage,
+										Formats: map[string]stream.ImageFormat{
+											"iso": stream.ImageFormat{
+												Disk: &stream.Artifact{
+													Location: fmt.Sprintf("%s/%s", svr.URL, ocReleaseImage),
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					}, nil
+				},
+			}
+			err = baseIso.Generate(dependencies)
 
-	GetIsoPluggable = func(archName string) (string, error) {
-		return "some-openshift-release.iso", nil
+			if tc.expectedError == "" {
+				assert.NoError(t, err)
+				assert.Regexp(t, tc.expectedBaseIsoFilename, baseIso.File.Filename)
+			} else {
+				assert.Equal(t, tc.expectedError, err.Error())
+			}
+		})
 	}
+}
+
+type mockRelease struct {
+	isoBaseVersion  string
+	baseIsoFileName string
+	baseIsoError    error
+}
+
+func (m *mockRelease) GetBaseIso(architecture string) (string, error) {
+	if m.baseIsoError != nil {
+		return "", m.baseIsoError
+	}
+	return m.baseIsoFileName, nil
+}
+
+func (m *mockRelease) GetBaseIsoVersion(architecture string) (string, error) {
+	return m.isoBaseVersion, nil
+}
+
+func (m *mockRelease) ExtractFile(image string, filename string, architecture string) ([]string, error) {
+	return []string{}, nil
+}
+
+func TestInfraBaseIso_GenerateOld(t *testing.T) {
 
 	parents := asset.Parents{}
 	manifests := &manifests.AgentManifests{}
diff --git a/pkg/asset/agent/image/ignition.go b/pkg/asset/agent/image/ignition.go
index f83a63fc7e..a22cb71a89 100644
--- a/pkg/asset/agent/image/ignition.go
+++ b/pkg/asset/agent/image/ignition.go
@@ -12,6 +12,7 @@ import (
 	"github.com/coreos/ignition/v2/config/util"
 	igntypes "github.com/coreos/ignition/v2/config/v3_2/types"
 	"github.com/coreos/stream-metadata-go/arch"
+	"github.com/coreos/stream-metadata-go/stream"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -32,7 +33,6 @@ import (
 	"github.com/openshift/installer/pkg/asset/ignition/bootstrap"
 	"github.com/openshift/installer/pkg/asset/password"
 	"github.com/openshift/installer/pkg/asset/tls"
-	"github.com/openshift/installer/pkg/rhcos"
 	"github.com/openshift/installer/pkg/types"
 	"github.com/openshift/installer/pkg/types/agent"
 	"github.com/openshift/installer/pkg/version"
@@ -144,6 +144,8 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 	numWorkers := 0
 	enabledServices := getDefaultEnabledServices()
 	openshiftVersion := ""
+	var err error
+	var streamGetter CoreOSBuildFetcher
 
 	switch agentWorkflow.Workflow {
 	case workflow.AgentWorkflowTypeInstall:
@@ -169,6 +171,7 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 		if err != nil {
 			return err
 		}
+		streamGetter = DefaultCoreOSStreamGetter
 
 	case workflow.AgentWorkflowTypeAddNodes:
 		// In the add-nodes workflow, every node will act independently from the others.
@@ -185,6 +188,9 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 		config.Storage.Files = append(config.Storage.Files, addNodesEnvFile)
 		// Version
 		openshiftVersion = clusterInfo.Version
+		streamGetter = func(ctx context.Context) (*stream.Stream, error) {
+			return clusterInfo.OSImage, nil
+		}
 
 	default:
 		return fmt.Errorf("AgentWorkflowType value not supported: %s", agentWorkflow.Workflow)
@@ -224,7 +230,7 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 	infraEnvID := uuid.New().String()
 	logrus.Debug("Generated random infra-env id ", infraEnvID)
 
-	osImage, err := getOSImagesInfoWithVersion(archName, openshiftVersion)
+	osImage, err := getOSImagesInfo(archName, openshiftVersion, streamGetter)
 	if err != nil {
 		return err
 	}
@@ -550,16 +556,8 @@ func addExtraManifests(config *igntypes.Config, extraManifests *manifests.ExtraM
 	return nil
 }
 
-func getOSImagesInfo(cpuArch string) (*models.OsImage, error) {
-	openshiftVersion, err := version.Version()
-	if err != nil {
-		return nil, err
-	}
-	return getOSImagesInfoWithVersion(cpuArch, openshiftVersion)
-}
-
-func getOSImagesInfoWithVersion(cpuArch string, openshiftVersion string) (*models.OsImage, error) {
-	st, err := rhcos.FetchCoreOSBuild(context.Background())
+func getOSImagesInfo(cpuArch string, openshiftVersion string, streamGetter CoreOSBuildFetcher) (*models.OsImage, error) {
+	st, err := streamGetter(context.Background())
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/asset/agent/image/releaseextract.go b/pkg/asset/agent/image/releaseextract.go
index efbb2467a4..55e766f8ba 100644
--- a/pkg/asset/agent/image/releaseextract.go
+++ b/pkg/asset/agent/image/releaseextract.go
@@ -26,7 +26,6 @@ import (
 	operatorv1alpha1 "github.com/openshift/api/operator/v1alpha1"
 	"github.com/openshift/installer/pkg/asset/agent"
 	"github.com/openshift/installer/pkg/asset/agent/mirror"
-	"github.com/openshift/installer/pkg/rhcos"
 	"github.com/openshift/installer/pkg/rhcos/cache"
 )
 
@@ -59,15 +58,17 @@ type release struct {
 	releaseImage string
 	pullSecret   string
 	mirrorConfig []mirror.RegistriesConfig
+	streamGetter CoreOSBuildFetcher
 }
 
-// NewRelease is used to set up the executor to run oc commands.
-func NewRelease(config Config, releaseImage string, pullSecret string, mirrorConfig []mirror.RegistriesConfig) Release {
+// NewRelease is used to set up the executor to run oc commands
+func NewRelease(config Config, releaseImage string, pullSecret string, mirrorConfig []mirror.RegistriesConfig, streamGetter CoreOSBuildFetcher) Release {
 	return &release{
 		config:       config,
 		releaseImage: releaseImage,
 		pullSecret:   pullSecret,
 		mirrorConfig: mirrorConfig,
+		streamGetter: streamGetter,
 	}
 }
 
@@ -258,13 +259,13 @@ func (r *release) extractFileFromImage(image, file, cacheDir string, architectur
 	return matches, nil
 }
 
-// Get hash from rhcos.json.
-func getHashFromInstaller(architecture string) (bool, string) {
+// Get hash from rhcos.json
+func (r *release) getHashFromInstaller(architecture string) (bool, string) {
 	// Get hash from metadata in the installer
 	ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second)
 	defer cancel()
 
-	st, err := rhcos.FetchCoreOSBuild(ctx)
+	st, err := r.streamGetter(ctx)
 	if err != nil {
 		return false, ""
 	}
@@ -307,7 +308,7 @@ func (r *release) verifyCacheFile(image, file, architecture string) (bool, error
 	fileSha := h.Sum(nil)
 
 	// Check if the hash of cached file matches hash in rhcos.json
-	found, rhcosSha := getHashFromInstaller(architecture)
+	found, rhcosSha := r.getHashFromInstaller(architecture)
 	if found && matchingHash(fileSha, rhcosSha) {
 		logrus.Debug("Found matching hash in installer metadata")
 		return true, nil
diff --git a/pkg/asset/agent/image/unconfigured_ignition.go b/pkg/asset/agent/image/unconfigured_ignition.go
index d33f368a02..98c50ac9fa 100644
--- a/pkg/asset/agent/image/unconfigured_ignition.go
+++ b/pkg/asset/agent/image/unconfigured_ignition.go
@@ -17,6 +17,7 @@ import (
 	"github.com/openshift/installer/pkg/asset/ignition"
 	"github.com/openshift/installer/pkg/asset/ignition/bootstrap"
 	"github.com/openshift/installer/pkg/types"
+	"github.com/openshift/installer/pkg/version"
 )
 
 const (
@@ -124,7 +125,11 @@ func (a *UnconfiguredIgnition) Generate(dependencies asset.Parents) error {
 	infraEnvID := uuid.New().String()
 	logrus.Debug("Generated random infra-env id ", infraEnvID)
 
-	osImage, err := getOSImagesInfo(archName)
+	openshiftVersion, err := version.Version()
+	if err != nil {
+		return err
+	}
+	osImage, err := getOSImagesInfo(archName, openshiftVersion, DefaultCoreOSStreamGetter)
 	if err != nil {
 		return err
 	}

From 298975a4df9852c7a8b10d2e19bb1ab46797fe19 Mon Sep 17 00:00:00 2001
From: Andrea Fasano <afasano@redhat.com>
Date: Wed, 6 Mar 2024 09:42:46 -0500
Subject: [PATCH 05/14] utlity script for building node-joiner binary

---
 hack/build-node-joiner.sh | 50 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100755 hack/build-node-joiner.sh

diff --git a/hack/build-node-joiner.sh b/hack/build-node-joiner.sh
new file mode 100755
index 0000000000..6acdfd92c6
--- /dev/null
+++ b/hack/build-node-joiner.sh
@@ -0,0 +1,50 @@
+#!/bin/sh
+
+set -ex
+
+# shellcheck disable=SC2068
+version() { IFS="."; printf "%03d%03d%03d\\n" $@; unset IFS;}
+
+minimum_go_version=1.21
+current_go_version=$(go version | cut -d " " -f 3)
+
+if [ "$(version "${current_go_version#go}")" -lt "$(version "$minimum_go_version")" ]; then
+     echo "Go version should be greater or equal to $minimum_go_version"
+     exit 1
+fi
+
+export CGO_ENABLED=0
+MODE="${MODE:-release}"
+
+GIT_COMMIT="${SOURCE_GIT_COMMIT:-$(git rev-parse --verify 'HEAD^{commit}')}"
+GIT_TAG="${BUILD_VERSION:-$(git describe --always --abbrev=40 --dirty)}"
+DEFAULT_ARCH="${DEFAULT_ARCH:-amd64}"
+GOFLAGS="${GOFLAGS:--mod=vendor}"
+GCFLAGS=""
+LDFLAGS="${LDFLAGS} -X github.com/openshift/installer/pkg/version.Raw=${GIT_TAG} -X github.com/openshift/installer/pkg/version.Commit=${GIT_COMMIT} -X github.com/openshift/installer/pkg/version.defaultArch=${DEFAULT_ARCH}"
+TAGS="${TAGS:-}"
+OUTPUT="${OUTPUT:-bin/node-joiner}"
+
+case "${MODE}" in
+release)
+	LDFLAGS="${LDFLAGS} -s -w"
+	TAGS="${TAGS} release"
+	;;
+dev)
+    GCFLAGS="${GCFLAGS} all=-N -l"
+	;;
+*)
+	echo "unrecognized mode: ${MODE}" >&2
+	exit 1
+esac
+
+if test "${SKIP_GENERATION}" != y
+then
+	# this step has to be run natively, even when cross-compiling
+	GOOS='' GOARCH='' go generate ./data
+fi
+
+echo "building node-joiner"
+
+# shellcheck disable=SC2086
+echo go build ${GOFLAGS} -gcflags "${GCFLAGS}" -ldflags "${LDFLAGS}" -tags "${TAGS}" -o "${OUTPUT}" ./cmd/node-joiner

From e5048da83d68eb22cc9fa15f6e26894c5a67187f Mon Sep 17 00:00:00 2001
From: Andrea Fasano <afasano@redhat.com>
Date: Wed, 6 Mar 2024 10:17:43 -0500
Subject: [PATCH 06/14] lint/fmt fixes

---
 .../agent/files/usr/local/bin/add-node.sh     | 10 ++--
 pkg/asset/agent/image/baseiso.go              |  9 ++--
 pkg/asset/agent/image/baseiso_test.go         | 46 ++++---------------
 pkg/asset/agent/image/ignition.go             |  8 ++--
 pkg/asset/agent/joiner/clusterinfo.go         |  4 +-
 5 files changed, 25 insertions(+), 52 deletions(-)

diff --git a/data/data/agent/files/usr/local/bin/add-node.sh b/data/data/agent/files/usr/local/bin/add-node.sh
index 49a3e324a8..aba855189e 100644
--- a/data/data/agent/files/usr/local/bin/add-node.sh
+++ b/data/data/agent/files/usr/local/bin/add-node.sh
@@ -22,14 +22,14 @@ status_issue="90_add-node"
 
 # For some reason the initial role patching doesn't seem to work properly
 echo "Patching host..."
-export HOST_ID=$(curl -s ${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts | jq -r '.[].id')
-curl -X PATCH -d '{"host_role":"worker"}' -H "Content-Type: application/json" ${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID}
+HOST_ID=$(curl -s "${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts" | jq -r '.[].id')
+curl -X PATCH -d '{"host_role":"worker"}' -H "Content-Type: application/json" "${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID}"
 
 # Wait for the current host to be ready
 host_ready=false
-while [[ host_ready == false ]]
+while [[ $host_ready == false ]]
 do
-    host_status=$(curl -s -S "${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID}" | jq -r .[].status)
+    host_status=$(curl -s -S "${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts" | jq -r ".[].status")
     if [[ "${host_status}" != "known" ]]; then
         printf '\\e{yellow}Waiting for the host to be ready' | set_issue "${status_issue}"
         sleep 10
@@ -43,5 +43,5 @@ clear_issue "${status_issue}"
 sleep 1m
 
 # Add the current host to the cluster
-curl -X POST -s -S ${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID}/actions/install
+curl -X POST -s -S "${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID}/actions/install"
 echo "Host installation started" 1>&2
diff --git a/pkg/asset/agent/image/baseiso.go b/pkg/asset/agent/image/baseiso.go
index d3cf08adc9..0e332bbe81 100644
--- a/pkg/asset/agent/image/baseiso.go
+++ b/pkg/asset/agent/image/baseiso.go
@@ -31,13 +31,13 @@ type BaseIso struct {
 	ocRelease    Release
 }
 
+// CoreOSBuildFetcher will be to used to switch the source of the coreos metadata.
 type CoreOSBuildFetcher func(ctx context.Context) (*stream.Stream, error)
 
 var (
-	baseIsoFilename           = ""
-	DefaultCoreOSStreamGetter = func(ctx context.Context) (*stream.Stream, error) {
-		return rhcos.FetchCoreOSBuild(ctx)
-	}
+	baseIsoFilename = ""
+	// DefaultCoreOSStreamGetter uses the pinned metadata.
+	DefaultCoreOSStreamGetter = rhcos.FetchCoreOSBuild
 )
 
 var _ asset.WritableAsset = (*BaseIso)(nil)
@@ -143,7 +143,6 @@ func (i *BaseIso) checkReleasePayloadBaseISOVersion(r Release, archName string)
 
 // Generate the baseIso
 func (i *BaseIso) Generate(dependencies asset.Parents) error {
-
 	var err error
 	var baseIsoFileName string
 
diff --git a/pkg/asset/agent/image/baseiso_test.go b/pkg/asset/agent/image/baseiso_test.go
index 1f92a893cd..06ab02f927 100644
--- a/pkg/asset/agent/image/baseiso_test.go
+++ b/pkg/asset/agent/image/baseiso_test.go
@@ -8,7 +8,6 @@ import (
 	"net/http/httptest"
 	"os"
 	"os/exec"
-	"path"
 	"testing"
 
 	"github.com/coreos/stream-metadata-go/stream"
@@ -18,22 +17,13 @@ import (
 	"github.com/openshift/assisted-service/api/v1beta1"
 	v1 "github.com/openshift/hive/apis/hive/v1"
 	"github.com/openshift/installer/pkg/asset"
-	"github.com/openshift/installer/pkg/asset/agent"
 	"github.com/openshift/installer/pkg/asset/agent/joiner"
 	"github.com/openshift/installer/pkg/asset/agent/manifests"
 	"github.com/openshift/installer/pkg/asset/agent/mirror"
 	"github.com/openshift/installer/pkg/asset/agent/workflow"
 )
 
-func setupEmbeddedResources(t *testing.T) {
-	workingDirectory, err := os.Getwd()
-	assert.NoError(t, err)
-	err = os.Chdir(path.Join(workingDirectory, "../../../../data"))
-	assert.NoError(t, err)
-}
-
 func TestBaseIso_Generate(t *testing.T) {
-	setupEmbeddedResources(t)
 	ocReleaseImage := "416.94.202402130130-0"
 	ocBaseIsoFilename := "openshift-4.16"
 
@@ -92,7 +82,7 @@ func TestBaseIso_Generate(t *testing.T) {
 				},
 				&mirror.RegistriesConf{},
 			},
-			getIsoError:             &exec.Error{"", exec.ErrNotFound},
+			getIsoError:             &exec.Error{},
 			expectedBaseIsoFilename: ocReleaseImage,
 		},
 	}
@@ -105,27 +95,27 @@ func TestBaseIso_Generate(t *testing.T) {
 			svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				// Answer with a fixed size randomly filled buffer
 				buffer := make([]byte, 1024)
-				rand.Read(buffer)
-				w.Write(buffer)
+				_, err := rand.Read(buffer)
+				assert.NoError(t, err)
+				_, err = w.Write(buffer)
+				assert.NoError(t, err)
 			}))
 			defer svr.Close()
 			// Creates a tmp folder to store the .cache downloaded images.
 			tmpPath, err := os.MkdirTemp("", "agent-baseiso-test")
 			assert.NoError(t, err)
 			previousXdgCacheHomeValue := os.Getenv("XDG_CACHE_HOME")
-			os.Setenv("XDG_CACHE_HOME", tmpPath)
+			t.Setenv("XDG_CACHE_HOME", tmpPath)
 			// Set the image override if defined
 			previousOpenshiftInstallOsImageOverrideValue := os.Getenv("OPENSHIFT_INSTALL_OS_IMAGE_OVERRIDE")
 			if tc.envVarOsImageOverrideValue != "" {
 				newOsImageOverride := fmt.Sprintf("%s/%s", svr.URL, tc.envVarOsImageOverrideValue)
-				os.Setenv("OPENSHIFT_INSTALL_OS_IMAGE_OVERRIDE", newOsImageOverride)
+				t.Setenv("OPENSHIFT_INSTALL_OS_IMAGE_OVERRIDE", newOsImageOverride)
 			}
 			// Cleanup on exit.
 			defer func() {
-				err := os.Setenv("XDG_CACHE_HOME", previousXdgCacheHomeValue)
-				assert.NoError(t, err)
-				err = os.Setenv("OPENSHIFT_INSTALL_OS_IMAGE_OVERRIDE", previousOpenshiftInstallOsImageOverrideValue)
-				assert.NoError(t, err)
+				t.Setenv("XDG_CACHE_HOME", previousXdgCacheHomeValue)
+				t.Setenv("OPENSHIFT_INSTALL_OS_IMAGE_OVERRIDE", previousOpenshiftInstallOsImageOverrideValue)
 				err = os.RemoveAll(tmpPath)
 				assert.NoError(t, err)
 			}()
@@ -144,7 +134,7 @@ func TestBaseIso_Generate(t *testing.T) {
 									"metal": {
 										Release: ocReleaseImage,
 										Formats: map[string]stream.ImageFormat{
-											"iso": stream.ImageFormat{
+											"iso": {
 												Disk: &stream.Artifact{
 													Location: fmt.Sprintf("%s/%s", svr.URL, ocReleaseImage),
 												},
@@ -189,19 +179,3 @@ func (m *mockRelease) GetBaseIsoVersion(architecture string) (string, error) {
 func (m *mockRelease) ExtractFile(image string, filename string, architecture string) ([]string, error) {
 	return []string{}, nil
 }
-
-func TestInfraBaseIso_GenerateOld(t *testing.T) {
-
-	parents := asset.Parents{}
-	manifests := &manifests.AgentManifests{}
-	installConfig := &agent.OptionalInstallConfig{}
-	parents.Add(manifests, installConfig)
-
-	asset := &BaseIso{}
-	err := asset.Generate(parents)
-	assert.NoError(t, err)
-
-	assert.NotEmpty(t, asset.Files())
-	baseIso := asset.Files()[0]
-	assert.Equal(t, baseIso.Filename, "some-openshift-release.iso")
-}
diff --git a/pkg/asset/agent/image/ignition.go b/pkg/asset/agent/image/ignition.go
index a22cb71a89..a4a86acbca 100644
--- a/pkg/asset/agent/image/ignition.go
+++ b/pkg/asset/agent/image/ignition.go
@@ -164,9 +164,9 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 		// Fetch the required number of master and worker nodes.
 		numMasters = agentManifests.AgentClusterInstall.Spec.ProvisionRequirements.ControlPlaneAgents
 		numWorkers = agentManifests.AgentClusterInstall.Spec.ProvisionRequirements.WorkerAgents
-		// Service
+		// Enable specific install services
 		enabledServices = append(enabledServices, "agent-register-cluster.service", "start-cluster-installation.service")
-		//Version
+		// Version is retrieved from the embedded data
 		openshiftVersion, err = version.Version()
 		if err != nil {
 			return err
@@ -181,12 +181,12 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 		// Fetch the required number of master and worker nodes.
 		numMasters = 0
 		numWorkers = len(addNodesConfig.Config.Hosts)
-		// Service
+		// Enable add-nodes specific services
 		enabledServices = append(enabledServices, "agent-import-cluster.service", "agent-add-node.service")
 		// Generate add-nodes.env file
 		addNodesEnvFile := ignition.FileFromString(addNodesEnvPath, "root", 0644, getAddNodesEnv(*clusterInfo))
 		config.Storage.Files = append(config.Storage.Files, addNodesEnvFile)
-		// Version
+		// Version matches the source cluster one
 		openshiftVersion = clusterInfo.Version
 		streamGetter = func(ctx context.Context) (*stream.Stream, error) {
 			return clusterInfo.OSImage, nil
diff --git a/pkg/asset/agent/joiner/clusterinfo.go b/pkg/asset/agent/joiner/clusterinfo.go
index fb9b3ae1fa..e5ffd5699a 100644
--- a/pkg/asset/agent/joiner/clusterinfo.go
+++ b/pkg/asset/agent/joiner/clusterinfo.go
@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/coreos/stream-metadata-go/arch"
+	"github.com/coreos/stream-metadata-go/stream"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -12,8 +14,6 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/yaml"
 
-	"github.com/coreos/stream-metadata-go/arch"
-	"github.com/coreos/stream-metadata-go/stream"
 	hiveext "github.com/openshift/assisted-service/api/hiveextension/v1beta1"
 	configclient "github.com/openshift/client-go/config/clientset/versioned"
 	"github.com/openshift/installer/pkg/asset"

From 0bf79d6245ed81619901d3f4816b7f9dd46777b0 Mon Sep 17 00:00:00 2001
From: Andrea Fasano <afasano@redhat.com>
Date: Mon, 11 Mar 2024 07:20:43 -0400
Subject: [PATCH 07/14] Fix build script

---
 hack/build-node-joiner.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/build-node-joiner.sh b/hack/build-node-joiner.sh
index 6acdfd92c6..8837b05d8f 100755
--- a/hack/build-node-joiner.sh
+++ b/hack/build-node-joiner.sh
@@ -47,4 +47,4 @@ fi
 echo "building node-joiner"
 
 # shellcheck disable=SC2086
-echo go build ${GOFLAGS} -gcflags "${GCFLAGS}" -ldflags "${LDFLAGS}" -tags "${TAGS}" -o "${OUTPUT}" ./cmd/node-joiner
+go build ${GOFLAGS} -gcflags "${GCFLAGS}" -ldflags "${LDFLAGS}" -tags "${TAGS}" -o "${OUTPUT}" ./cmd/node-joiner

From 015f2287090c2007d874b85bd8cef7e97a0e0031 Mon Sep 17 00:00:00 2001
From: Andrea Fasano <afasano@redhat.com>
Date: Mon, 11 Mar 2024 07:32:51 -0400
Subject: [PATCH 08/14] add log levels support

---
 cmd/node-joiner/main.go | 41 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 40 insertions(+), 1 deletion(-)

diff --git a/cmd/node-joiner/main.go b/cmd/node-joiner/main.go
index baf6a58210..84116634b2 100644
--- a/cmd/node-joiner/main.go
+++ b/cmd/node-joiner/main.go
@@ -1,9 +1,15 @@
 package main
 
 import (
+	"fmt"
+	"io"
+	"os"
+
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	terminal "golang.org/x/term"
 
+	"github.com/openshift/installer/cmd/openshift-install/command"
 	"github.com/openshift/installer/pkg/nodejoiner"
 )
 
@@ -33,10 +39,12 @@ func main() {
 	}
 
 	rootCmd := &cobra.Command{
-		Use: "node-joiner",
+		Use:              "node-joiner",
+		PersistentPreRun: runRootCmd,
 	}
 	rootCmd.PersistentFlags().String("kubeconfig", "", "Path to the kubeconfig file.")
 	rootCmd.PersistentFlags().String("dir", ".", "assets directory")
+	rootCmd.PersistentFlags().String("log-level", "info", "log level (e.g. \"debug | info | warn | error\")")
 
 	rootCmd.AddCommand(nodesAddCmd)
 	rootCmd.AddCommand(nodesMonitorCmd)
@@ -44,3 +52,34 @@ func main() {
 		logrus.Fatal(err)
 	}
 }
+
+func runRootCmd(cmd *cobra.Command, args []string) {
+	logrus.SetOutput(io.Discard)
+	logrus.SetLevel(logrus.TraceLevel)
+
+	logLevel, err := cmd.Flags().GetString("log-level")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	level, err := logrus.ParseLevel(logLevel)
+	if err != nil {
+		level = logrus.InfoLevel
+	}
+
+	logrus.AddHook(command.NewFileHookWithNewlineTruncate(os.Stderr, level, &logrus.TextFormatter{
+		// Setting ForceColors is necessary because logrus.TextFormatter determines
+		// whether or not to enable colors by looking at the output of the logger.
+		// In this case, the output is io.Discard, which is not a terminal.
+		// Overriding it here allows the same check to be done, but against the
+		// hook's output instead of the logger's output.
+		ForceColors:            terminal.IsTerminal(int(os.Stderr.Fd())),
+		DisableTimestamp:       true,
+		DisableLevelTruncation: true,
+		DisableQuote:           true,
+	}))
+
+	if err != nil {
+		logrus.Fatal(fmt.Errorf("invalid log-level: %w", err))
+	}
+}

From abf18528ff01122fa890da1f8e146562798972b0 Mon Sep 17 00:00:00 2001
From: Andrea Fasano <afasano@redhat.com>
Date: Mon, 11 Mar 2024 09:25:03 -0400
Subject: [PATCH 09/14] minor updates

---
 pkg/asset/agent/image/ignition.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/pkg/asset/agent/image/ignition.go b/pkg/asset/agent/image/ignition.go
index a4a86acbca..53b242d811 100644
--- a/pkg/asset/agent/image/ignition.go
+++ b/pkg/asset/agent/image/ignition.go
@@ -178,7 +178,9 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 		a.RendezvousIP = "127.0.0.1"
 		// Reuse the existing cluster name.
 		clusterName = clusterInfo.ClusterName
-		// Fetch the required number of master and worker nodes.
+		// Fetch the required number of master and worker nodes. Currently only adding workers
+		// is supported, so forcing the expected number of masters to zero, and assuming implcitly
+		// that all the hosts defined are workers.
 		numMasters = 0
 		numWorkers = len(addNodesConfig.Config.Hosts)
 		// Enable add-nodes specific services
@@ -360,7 +362,7 @@ func addBootstrapScripts(config *igntypes.Config, releaseImage string) (err erro
 
 func getTemplateData(name, pullSecret, releaseImageList, releaseImage,
 	releaseImageMirror string, haveMirrorConfig bool, publicContainerRegistries string,
-	numMasters int, numWorkers int,
+	numMasters, numWorkers int,
 	infraEnvID string,
 	osImage *models.OsImage,
 	proxy *v1beta1.Proxy,

From 13c8db42872aa96745b668061062d4a850ce1c00 Mon Sep 17 00:00:00 2001
From: Andrea Fasano <afasano@redhat.com>
Date: Tue, 12 Mar 2024 09:52:15 -0400
Subject: [PATCH 10/14] inject ClusterInfo into NMStateConfig asset

---
 pkg/asset/agent/manifests/common.go           |  4 --
 pkg/asset/agent/manifests/nmstateconfig.go    | 29 +++++----
 .../agent/manifests/nmstateconfig_test.go     | 59 +++++++++++++++++--
 3 files changed, 74 insertions(+), 18 deletions(-)

diff --git a/pkg/asset/agent/manifests/common.go b/pkg/asset/agent/manifests/common.go
index b42b96f988..37573aaf2b 100644
--- a/pkg/asset/agent/manifests/common.go
+++ b/pkg/asset/agent/manifests/common.go
@@ -27,10 +27,6 @@ func getProxy(proxy *types.Proxy) *aiv1beta1.Proxy {
 	}
 }
 
-func getNMStateConfigName(ic *agent.OptionalInstallConfig) string {
-	return ic.ClusterName()
-}
-
 func getNMStateConfigLabels(clusterName string) map[string]string {
 	return map[string]string{
 		"infraenvs.agent-install.openshift.io": clusterName,
diff --git a/pkg/asset/agent/manifests/nmstateconfig.go b/pkg/asset/agent/manifests/nmstateconfig.go
index df578a2db8..605a44d862 100644
--- a/pkg/asset/agent/manifests/nmstateconfig.go
+++ b/pkg/asset/agent/manifests/nmstateconfig.go
@@ -80,21 +80,30 @@ func (n *NMStateConfig) Generate(dependencies asset.Parents) error {
 	installConfig := &agent.OptionalInstallConfig{}
 	dependencies.Get(agentHosts, installConfig, agentWorkflow, clusterInfo)
 
-	// Not required for the add nodes workflow.
-	if agentWorkflow.Workflow == workflow.AgentWorkflowTypeAddNodes {
-		return nil
-	}
-
 	staticNetworkConfig := []*models.HostStaticNetworkConfig{}
 	nmStateConfigs := []*aiv1beta1.NMStateConfig{}
 	var data string
 	var isNetworkConfigAvailable bool
+	var clusterName, clusterNamespace string
 
 	if len(agentHosts.Hosts) == 0 {
 		return nil
 	}
-	if err := validateHostCount(installConfig.Config, agentHosts); err != nil {
-		return err
+
+	switch agentWorkflow.Workflow {
+	case workflow.AgentWorkflowTypeInstall:
+		if err := validateHostCount(installConfig.Config, agentHosts); err != nil {
+			return err
+		}
+		clusterName = installConfig.ClusterName()
+		clusterNamespace = installConfig.ClusterNamespace()
+
+	case workflow.AgentWorkflowTypeAddNodes:
+		clusterName = clusterInfo.ClusterName
+		clusterNamespace = clusterInfo.Namespace
+
+	default:
+		return fmt.Errorf("AgentWorkflowType value not supported: %s", agentWorkflow.Workflow)
 	}
 
 	for i, host := range agentHosts.Hosts {
@@ -107,9 +116,9 @@ func (n *NMStateConfig) Generate(dependencies asset.Parents) error {
 					APIVersion: "agent-install.openshift.io/v1beta1",
 				},
 				ObjectMeta: metav1.ObjectMeta{
-					Name:      fmt.Sprintf(getNMStateConfigName(installConfig)+"-%d", i),
-					Namespace: installConfig.ClusterNamespace(),
-					Labels:    getNMStateConfigLabels(installConfig.ClusterName()),
+					Name:      fmt.Sprintf("%s-%d", clusterName, i),
+					Namespace: clusterNamespace,
+					Labels:    getNMStateConfigLabels(clusterName),
 				},
 				Spec: aiv1beta1.NMStateConfigSpec{
 					NetConfig: aiv1beta1.NetConfig{
diff --git a/pkg/asset/agent/manifests/nmstateconfig_test.go b/pkg/asset/agent/manifests/nmstateconfig_test.go
index 4463bba6e6..2211414e0a 100644
--- a/pkg/asset/agent/manifests/nmstateconfig_test.go
+++ b/pkg/asset/agent/manifests/nmstateconfig_test.go
@@ -14,6 +14,7 @@ import (
 	aiv1beta1 "github.com/openshift/assisted-service/api/v1beta1"
 	"github.com/openshift/assisted-service/models"
 	"github.com/openshift/installer/pkg/asset"
+	agentconfig "github.com/openshift/installer/pkg/asset/agent"
 	"github.com/openshift/installer/pkg/asset/agent/joiner"
 	"github.com/openshift/installer/pkg/asset/agent/workflow"
 	"github.com/openshift/installer/pkg/asset/mock"
@@ -28,6 +29,56 @@ func TestNMStateConfig_Generate(t *testing.T) {
 		expectedConfig     []*aiv1beta1.NMStateConfig
 		expectedError      string
 	}{
+		{
+			name: "add-nodes workflow",
+			dependencies: []asset.Asset{
+				&workflow.AgentWorkflow{Workflow: workflow.AgentWorkflowTypeAddNodes},
+				&joiner.ClusterInfo{},
+				getAgentHostsNoHosts(),
+				&agentconfig.OptionalInstallConfig{},
+			},
+			requiresNmstatectl: false,
+			expectedConfig:     nil,
+			expectedError:      "",
+		},
+		{
+			name: "add-nodes workflow - agentHosts with some hosts without networkconfig",
+			dependencies: []asset.Asset{
+				&workflow.AgentWorkflow{Workflow: workflow.AgentWorkflowTypeAddNodes},
+				&joiner.ClusterInfo{
+					Namespace:   "cluster0",
+					ClusterName: "ostest",
+				},
+				getAgentHostsWithSomeHostsWithoutNetworkConfig(),
+				&agentconfig.OptionalInstallConfig{},
+			},
+			requiresNmstatectl: true,
+			expectedConfig: []*aiv1beta1.NMStateConfig{
+				{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       "NMStateConfig",
+						APIVersion: "agent-install.openshift.io/v1beta1",
+					},
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "ostest-0",
+						Namespace: "cluster0",
+						Labels:    getNMStateConfigLabels("ostest"),
+					},
+					Spec: aiv1beta1.NMStateConfigSpec{
+						Interfaces: []*aiv1beta1.Interface{
+							{
+								Name:       "enp2t0",
+								MacAddress: "98:af:65:a5:8d:02",
+							},
+						},
+						NetConfig: aiv1beta1.NetConfig{
+							Raw: unmarshalJSON([]byte(rawNMStateConfigNoIP)),
+						},
+					},
+				},
+			},
+			expectedError: "",
+		},
 		{
 			name: "agentHosts does not contain networkConfig",
 			dependencies: []asset.Asset{
@@ -56,7 +107,7 @@ func TestNMStateConfig_Generate(t *testing.T) {
 						APIVersion: "agent-install.openshift.io/v1beta1",
 					},
 					ObjectMeta: metav1.ObjectMeta{
-						Name:      fmt.Sprint(getNMStateConfigName(getValidOptionalInstallConfig()), "-0"),
+						Name:      fmt.Sprint(getValidOptionalInstallConfig().ClusterName(), "-0"),
 						Namespace: getValidOptionalInstallConfig().ClusterNamespace(),
 						Labels:    getNMStateConfigLabels(getValidOptionalInstallConfig().ClusterName()),
 					},
@@ -91,7 +142,7 @@ func TestNMStateConfig_Generate(t *testing.T) {
 						APIVersion: "agent-install.openshift.io/v1beta1",
 					},
 					ObjectMeta: metav1.ObjectMeta{
-						Name:      fmt.Sprint(getNMStateConfigName(getValidOptionalInstallConfig()), "-0"),
+						Name:      fmt.Sprint(getValidOptionalInstallConfig().ClusterName(), "-0"),
 						Namespace: getValidOptionalInstallConfig().ClusterNamespace(),
 						Labels:    getNMStateConfigLabels(getValidOptionalInstallConfig().ClusterName()),
 					},
@@ -117,7 +168,7 @@ func TestNMStateConfig_Generate(t *testing.T) {
 						APIVersion: "agent-install.openshift.io/v1beta1",
 					},
 					ObjectMeta: metav1.ObjectMeta{
-						Name:      fmt.Sprint(getNMStateConfigName(getValidOptionalInstallConfig()), "-1"),
+						Name:      fmt.Sprint(getValidOptionalInstallConfig().ClusterName(), "-1"),
 						Namespace: getValidOptionalInstallConfig().ClusterNamespace(),
 						Labels:    getNMStateConfigLabels(getValidOptionalInstallConfig().ClusterName()),
 					},
@@ -139,7 +190,7 @@ func TestNMStateConfig_Generate(t *testing.T) {
 						APIVersion: "agent-install.openshift.io/v1beta1",
 					},
 					ObjectMeta: metav1.ObjectMeta{
-						Name:      fmt.Sprint(getNMStateConfigName(getValidOptionalInstallConfig()), "-2"),
+						Name:      fmt.Sprint(getValidOptionalInstallConfig().ClusterName(), "-2"),
 						Namespace: getValidOptionalInstallConfig().ClusterNamespace(),
 						Labels:    getNMStateConfigLabels(getValidOptionalInstallConfig().ClusterName()),
 					},

From 8ea9b936df1aaed30e9a1d231509a1f19fe6c2d3 Mon Sep 17 00:00:00 2001
From: Andrea Fasano <afasano@redhat.com>
Date: Tue, 12 Mar 2024 10:47:44 -0400
Subject: [PATCH 11/14] use different iso name for node-joiner add-nodes
 command

---
 pkg/asset/agent/image/agentimage.go | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/pkg/asset/agent/image/agentimage.go b/pkg/asset/agent/image/agentimage.go
index 4c4e9dc826..6bc14b6da0 100644
--- a/pkg/asset/agent/image/agentimage.go
+++ b/pkg/asset/agent/image/agentimage.go
@@ -20,8 +20,9 @@ import (
 )
 
 const (
-	agentISOFilename    = "agent.%s.iso"
-	iso9660Level1ExtLen = 3
+	agentISOFilename         = "agent.%s.iso"
+	agentAddNodesISOFilename = "agent-addnodes.%s.iso"
+	iso9660Level1ExtLen      = 3
 )
 
 // AgentImage is an asset that generates the bootable image used to install clusters.
@@ -34,6 +35,7 @@ type AgentImage struct {
 	rootFSURL            string
 	bootArtifactsBaseURL string
 	platform             hiveext.PlatformType
+	isoFilename          string
 }
 
 var _ asset.WritableAsset = (*AgentImage)(nil)
@@ -61,9 +63,11 @@ func (a *AgentImage) Generate(dependencies asset.Parents) error {
 	switch agentWorkflow.Workflow {
 	case workflow.AgentWorkflowTypeInstall:
 		a.platform = agentManifests.AgentClusterInstall.Spec.PlatformType
+		a.isoFilename = agentISOFilename
 
 	case workflow.AgentWorkflowTypeAddNodes:
 		a.platform = clusterInfo.PlatformType
+		a.isoFilename = agentAddNodesISOFilename
 
 	default:
 		return fmt.Errorf("AgentWorkflowType value not supported: %s", agentWorkflow.Workflow)
@@ -239,7 +243,7 @@ func (a *AgentImage) PersistToFile(directory string) error {
 		return errors.New("cannot generate ISO image due to configuration errors")
 	}
 
-	agentIsoFile := filepath.Join(directory, fmt.Sprintf(agentISOFilename, a.cpuArch))
+	agentIsoFile := filepath.Join(directory, fmt.Sprintf(a.isoFilename, a.cpuArch))
 
 	// Remove symlink if it exists
 	os.Remove(agentIsoFile)

From 28fdc6e9763935a65a82c2e437ad1e14940d26ea Mon Sep 17 00:00:00 2001
From: Andrea Fasano <afasano@redhat.com>
Date: Fri, 15 Mar 2024 10:12:12 -0400
Subject: [PATCH 12/14] various add-node.sh fixes

- removed role patch as not required
- managed error for host install call
---
 .../agent/files/usr/local/bin/add-node.sh     | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/data/data/agent/files/usr/local/bin/add-node.sh b/data/data/agent/files/usr/local/bin/add-node.sh
index aba855189e..fccd683de4 100644
--- a/data/data/agent/files/usr/local/bin/add-node.sh
+++ b/data/data/agent/files/usr/local/bin/add-node.sh
@@ -20,11 +20,6 @@ printf '\nInfra env id is %s\n' "${INFRA_ENV_ID}" 1>&2
 
 status_issue="90_add-node"
 
-# For some reason the initial role patching doesn't seem to work properly
-echo "Patching host..."
-HOST_ID=$(curl -s "${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts" | jq -r '.[].id')
-curl -X PATCH -d '{"host_role":"worker"}' -H "Content-Type: application/json" "${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID}"
-
 # Wait for the current host to be ready
 host_ready=false
 while [[ $host_ready == false ]]
@@ -38,10 +33,16 @@ do
     fi
 done
 
+HOST_ID=$(curl -s "${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts" | jq -r '.[].id')
+printf '\nHost %s is ready for installation\n' "${HOST_ID}" 1>&2
 clear_issue "${status_issue}"
 
-sleep 1m
-
 # Add the current host to the cluster
-curl -X POST -s -S "${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID}/actions/install"
-echo "Host installation started" 1>&2
+res=$(curl -X POST -s -S "${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID}/actions/install")
+code=$(echo "$res" | jq -r '.code')
+message=$(echo "$res" | jq -r '.message')
+if [[ $res = "200" ]]; then 
+    printf "\nHost installation started\n" 1>&2
+else
+    printf '\nHost installation failed: %s\n' "${message}" 1>&2
+fi

From c1ac2378b85514b74f856ba453f620093bbeb96e Mon Sep 17 00:00:00 2001
From: Andrea Fasano <afasano@redhat.com>
Date: Fri, 29 Mar 2024 09:43:50 -0400
Subject: [PATCH 13/14] support adding multiple hosts

---
 data/data/agent/systemd/units/apply-host-config.service | 2 +-
 pkg/asset/agent/image/ignition.go                       | 7 ++++---
 pkg/asset/agent/image/ignition_test.go                  | 4 ++--
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/data/data/agent/systemd/units/apply-host-config.service b/data/data/agent/systemd/units/apply-host-config.service
index 5077ed9eab..95cc8e5633 100644
--- a/data/data/agent/systemd/units/apply-host-config.service
+++ b/data/data/agent/systemd/units/apply-host-config.service
@@ -14,7 +14,7 @@ EnvironmentFile=/usr/local/share/assisted-service/assisted-service.env
 ExecStartPre=/bin/rm -f %t/%n.ctr-id
 ExecStartPre=/bin/mkdir -p %t/agent-installer /etc/assisted/hostconfig
 ExecStartPre=/usr/local/bin/wait-for-assisted-service.sh
-ExecStart=podman run --net host --cidfile=%t/%n.ctr-id --cgroups=no-conmon --log-driver=journald --restart=on-failure:10 --pod-id-file=%t/assisted-service-pod.pod-id --replace --name=apply-host-config -v /etc/assisted/hostconfig:/etc/assisted/hostconfig -v %t/agent-installer:/var/run/agent-installer:z --env SERVICE_BASE_URL --env INFRA_ENV_ID $SERVICE_IMAGE /usr/local/bin/agent-installer-client configure
+ExecStart=podman run --net host --cidfile=%t/%n.ctr-id --cgroups=no-conmon --log-driver=journald --restart=on-failure:10 --pod-id-file=%t/assisted-service-pod.pod-id --replace --name=apply-host-config -v /etc/assisted/hostconfig:/etc/assisted/hostconfig -v %t/agent-installer:/var/run/agent-installer:z --env SERVICE_BASE_URL --env INFRA_ENV_ID --env WORKFLOW_TYPE $SERVICE_IMAGE /usr/local/bin/agent-installer-client configure
 ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id
 ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id
 
diff --git a/pkg/asset/agent/image/ignition.go b/pkg/asset/agent/image/ignition.go
index 53b242d811..17f761484c 100644
--- a/pkg/asset/agent/image/ignition.go
+++ b/pkg/asset/agent/image/ignition.go
@@ -262,7 +262,7 @@ func (a *Ignition) Generate(dependencies asset.Parents) error {
 
 	rendezvousHostFile := ignition.FileFromString(rendezvousHostEnvPath,
 		"root", 0644,
-		getRendezvousHostEnv(agentTemplateData.ServiceProtocol, a.RendezvousIP))
+		getRendezvousHostEnv(agentTemplateData.ServiceProtocol, a.RendezvousIP, agentWorkflow.Workflow))
 	config.Storage.Files = append(config.Storage.Files, rendezvousHostFile)
 
 	err = addBootstrapScripts(&config, agentManifests.ClusterImageSet.Spec.ReleaseImage)
@@ -390,7 +390,7 @@ func getTemplateData(name, pullSecret, releaseImageList, releaseImage,
 	}
 }
 
-func getRendezvousHostEnv(serviceProtocol, nodeZeroIP string) string {
+func getRendezvousHostEnv(serviceProtocol, nodeZeroIP string, workflowType workflow.AgentWorkflowType) string {
 	serviceBaseURL := url.URL{
 		Scheme: serviceProtocol,
 		Host:   net.JoinHostPort(nodeZeroIP, "8090"),
@@ -405,7 +405,8 @@ func getRendezvousHostEnv(serviceProtocol, nodeZeroIP string) string {
 	return fmt.Sprintf(`NODE_ZERO_IP=%s
 SERVICE_BASE_URL=%s
 IMAGE_SERVICE_BASE_URL=%s
-`, nodeZeroIP, serviceBaseURL.String(), imageServiceBaseURL.String())
+WORKFLOW_TYPE=%s
+`, nodeZeroIP, serviceBaseURL.String(), imageServiceBaseURL.String(), workflowType)
 }
 
 func getAddNodesEnv(clusterInfo joiner.ClusterInfo) string {
diff --git a/pkg/asset/agent/image/ignition_test.go b/pkg/asset/agent/image/ignition_test.go
index 82afa6a5f2..e7884a048e 100644
--- a/pkg/asset/agent/image/ignition_test.go
+++ b/pkg/asset/agent/image/ignition_test.go
@@ -112,9 +112,9 @@ func TestIgnition_getTemplateData(t *testing.T) {
 
 func TestIgnition_getRendezvousHostEnv(t *testing.T) {
 	nodeZeroIP := "2001:db8::dead:beef"
-	rendezvousHostEnv := getRendezvousHostEnv("http", nodeZeroIP)
+	rendezvousHostEnv := getRendezvousHostEnv("http", nodeZeroIP, workflow.AgentWorkflowTypeInstall)
 	assert.Equal(t,
-		"NODE_ZERO_IP="+nodeZeroIP+"\nSERVICE_BASE_URL=http://["+nodeZeroIP+"]:8090/\nIMAGE_SERVICE_BASE_URL=http://["+nodeZeroIP+"]:8888/\n",
+		"NODE_ZERO_IP="+nodeZeroIP+"\nSERVICE_BASE_URL=http://["+nodeZeroIP+"]:8090/\nIMAGE_SERVICE_BASE_URL=http://["+nodeZeroIP+"]:8888/\nWORKFLOW_TYPE=install\n",
 		rendezvousHostEnv)
 }
 

From cac843c18d618d740de6bfc509cb2376d9911e69 Mon Sep 17 00:00:00 2001
From: Andrea Fasano <afasano@redhat.com>
Date: Fri, 29 Mar 2024 12:38:35 -0400
Subject: [PATCH 14/14] additional lint/script fixes

---
 data/data/agent/files/usr/local/bin/add-node.sh | 11 +++++------
 pkg/asset/agent/image/releaseextract.go         |  4 ++--
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/data/data/agent/files/usr/local/bin/add-node.sh b/data/data/agent/files/usr/local/bin/add-node.sh
index fccd683de4..c0d4aa84f0 100644
--- a/data/data/agent/files/usr/local/bin/add-node.sh
+++ b/data/data/agent/files/usr/local/bin/add-node.sh
@@ -38,11 +38,10 @@ printf '\nHost %s is ready for installation\n' "${HOST_ID}" 1>&2
 clear_issue "${status_issue}"
 
 # Add the current host to the cluster
-res=$(curl -X POST -s -S "${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID}/actions/install")
-code=$(echo "$res" | jq -r '.code')
-message=$(echo "$res" | jq -r '.message')
-if [[ $res = "200" ]]; then 
-    printf "\nHost installation started\n" 1>&2
+res=$(curl -X POST -s -S -w "%{http_code}\\n" -o /dev/null "${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID}/actions/install")
+if [[ $res = "202" ]]; then 
+    printf '\nHost installation started\n' 1>&2
 else
-    printf '\nHost installation failed: %s\n' "${message}" 1>&2
+    printf '\nHost installation failed\n' 1>&2
+    exit 1
 fi
diff --git a/pkg/asset/agent/image/releaseextract.go b/pkg/asset/agent/image/releaseextract.go
index 55e766f8ba..3ab6bf6d30 100644
--- a/pkg/asset/agent/image/releaseextract.go
+++ b/pkg/asset/agent/image/releaseextract.go
@@ -61,7 +61,7 @@ type release struct {
 	streamGetter CoreOSBuildFetcher
 }
 
-// NewRelease is used to set up the executor to run oc commands
+// NewRelease is used to set up the executor to run oc commands.
 func NewRelease(config Config, releaseImage string, pullSecret string, mirrorConfig []mirror.RegistriesConfig, streamGetter CoreOSBuildFetcher) Release {
 	return &release{
 		config:       config,
@@ -259,7 +259,7 @@ func (r *release) extractFileFromImage(image, file, cacheDir string, architectur
 	return matches, nil
 }
 
-// Get hash from rhcos.json
+// Get hash from rhcos.json.
 func (r *release) getHashFromInstaller(architecture string) (bool, string) {
 	// Get hash from metadata in the installer
 	ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second)