From a58bf9646595553a1d040d8f763d3fe37b8a71e5 Mon Sep 17 00:00:00 2001 From: Andrea Fasano Date: Tue, 5 Sep 2023 08:53:43 -0400 Subject: [PATCH 1/3] agent: day2 add node spike --- cmd/openshift-install/agent.go | 14 ++++- .../files/usr/local/bin/import-cluster.sh | 11 ++++ .../local/bin/start-cluster-installation.sh | 58 +++++++++++------- pkg/asset/agent/image/agentimage.go | 15 ++++- pkg/asset/agent/image/ignition.go | 51 ++++++++++++---- pkg/asset/agent/image/ignition_test.go | 4 +- .../agent/manifests/agentclusterinstall.go | 8 ++- pkg/asset/agent/manifests/agentpullsecret.go | 24 +++++++- .../agent/manifests/clusterdeployment.go | 8 ++- pkg/asset/agent/manifests/infraenv.go | 56 +++++++++++++---- pkg/asset/agent/manifests/kubeconfigfile.go | 61 +++++++++++++++++++ pkg/asset/agent/manifests/nmstateconfig.go | 28 +++++++-- pkg/asset/agent/manifests/plainpullsecret.go | 49 +++++++++++++++ pkg/asset/agent/manifests/sshpublickey.go | 54 ++++++++++++++++ 14 files changed, 381 insertions(+), 60 deletions(-) create mode 100644 data/data/agent/files/usr/local/bin/import-cluster.sh create mode 100644 pkg/asset/agent/manifests/kubeconfigfile.go create mode 100644 pkg/asset/agent/manifests/plainpullsecret.go create mode 100644 pkg/asset/agent/manifests/sshpublickey.go diff --git a/cmd/openshift-install/agent.go b/cmd/openshift-install/agent.go index eb35c3ef0a3..e17cfc1387c 100644 --- a/cmd/openshift-install/agent.go +++ b/cmd/openshift-install/agent.go @@ -112,7 +112,19 @@ var ( }, } - agentTargets = []target{agentConfigTarget, agentManifestsTarget, agentImageTarget, agentPXEFilesTarget, agentConfigImageTarget, agentUnconfiguredIgnitionTarget} + agentImageAddNodesTarget = target{ + name: "Agent ISO Image for adding nodes", + command: &cobra.Command{ + Use: "nodes-image", + Short: "Generates a bootable image containing all the information needed to added one or more nodes to an already existing cluster", + Args: cobra.ExactArgs(0), + }, + assets: []asset.WritableAsset{ + &image.AgentImage{}, + }, + } + + agentTargets = []target{agentConfigTarget, agentManifestsTarget, agentImageTarget, agentPXEFilesTarget, agentConfigImageTarget, agentUnconfiguredIgnitionTarget, agentImageAddNodesTarget} ) func newAgentCreateCmd() *cobra.Command { diff --git a/data/data/agent/files/usr/local/bin/import-cluster.sh b/data/data/agent/files/usr/local/bin/import-cluster.sh new file mode 100644 index 00000000000..782da71d5b9 --- /dev/null +++ b/data/data/agent/files/usr/local/bin/import-cluster.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +echo "Waiting for assisted-service to be ready" +until curl --output /dev/null --silent --fail "${SERVICE_BASE_URL}/api/assisted-install/v2/infra-envs"; do + printf '.' + sleep 5 +done + +echo "Importing cluster from ${API_VIP_DNSNAME}" +curl -d '{"name":"abi", "api_vip_dnsname":"'"${API_VIP_DNSNAME}"'", "openshift_cluster_id":"764eb48f-8ffb-4126-b7ed-0ca746365654"}' -H "Content-Type: application/json" -X POST "${SERVICE_BASE_URL}/api/assisted-install/v2/clusters/import" \ No newline at end of file diff --git a/data/data/agent/files/usr/local/bin/start-cluster-installation.sh b/data/data/agent/files/usr/local/bin/start-cluster-installation.sh index ef6ad7954ac..bd7e1f6228e 100644 --- a/data/data/agent/files/usr/local/bin/start-cluster-installation.sh +++ b/data/data/agent/files/usr/local/bin/start-cluster-installation.sh @@ -48,6 +48,12 @@ num_known_hosts() { while [[ "${total_required_nodes}" != $(num_known_hosts) ]] do + + # For some reason the initial role patching doesn't seem to work properly, to be reviewed + echo "Patching host..." + export HOST_ID=$(curl -s ${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts | jq -r '.[].id') + curl -X PATCH -d '{"host_role":"worker"}' -H "Content-Type: application/json" ${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID} + echo "Waiting for hosts to become ready for cluster installation..." 1>&2 sleep 10 done @@ -55,26 +61,32 @@ done echo "All ${total_required_nodes} hosts are ready." 1>&2 clear_issue "${status_issue}" -while [[ "${cluster_status}" != "installed" ]] -do - sleep 5 - cluster_status=$(curl -s -S "${BASE_URL}/clusters" | jq -r .[].status) - echo "Cluster status: ${cluster_status}" 1>&2 - # Start the cluster install, if it transitions back to Ready due to a failure, - # then it will be restarted - case "${cluster_status}" in - "ready") - echo "Starting cluster installation..." 1>&2 - curl -s -S -X POST "${BASE_URL}/clusters/${cluster_id}/actions/install" \ - -H 'accept: application/json' \ - -d '' - echo "Cluster installation started" 1>&2 - ;& - "installed" | "preparing-for-installation" | "installing") - printf '\\e{lightgreen}Cluster installation in progress\\e{reset}' | set_issue "${status_issue}" - ;; - *) - printf '\\e{lightred}Installation cannot proceed:\\e{reset} Cluster status: %s' "${cluster_status}" | set_issue "${status_issue}" - ;; - esac -done +sleep 1m + +# Install the host +echo "Installing host..." +curl -X POST ${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID}/actions/install + +# while [[ "${cluster_status}" != "installed" ]] +# do +# sleep 5 +# cluster_status=$(curl -s -S "${BASE_URL}/clusters" | jq -r .[].status) +# echo "Cluster status: ${cluster_status}" 1>&2 +# # Start the cluster install, if it transitions back to Ready due to a failure, +# # then it will be restarted +# case "${cluster_status}" in +# "ready") +# echo "Starting cluster installation..." 1>&2 +# curl -s -S -X POST "${BASE_URL}/clusters/${cluster_id}/actions/install" \ +# -H 'accept: application/json' \ +# -d '' +# echo "Cluster installation started" 1>&2 +# ;& +# "installed" | "preparing-for-installation" | "installing") +# printf '\\e{lightgreen}Cluster installation in progress\\e{reset}' | set_issue "${status_issue}" +# ;; +# *) +# printf '\\e{lightred}Installation cannot proceed:\\e{reset} Cluster status: %s' "${cluster_status}" | set_issue "${status_issue}" +# ;; +# esac +# done diff --git a/pkg/asset/agent/image/agentimage.go b/pkg/asset/agent/image/agentimage.go index 4b759d3d534..904d8be9070 100644 --- a/pkg/asset/agent/image/agentimage.go +++ b/pkg/asset/agent/image/agentimage.go @@ -32,6 +32,7 @@ type AgentImage struct { rootFSURL string bootArtifactsBaseURL string platform hiveext.PlatformType + kubeconfig *manifests.KubeConfigFile } var _ asset.WritableAsset = (*AgentImage)(nil) @@ -42,6 +43,7 @@ func (a *AgentImage) Dependencies() []asset.Asset { &AgentArtifacts{}, &manifests.AgentManifests{}, &BaseIso{}, + &manifests.KubeConfigFile{}, } } @@ -50,13 +52,15 @@ func (a *AgentImage) Generate(dependencies asset.Parents) error { agentArtifacts := &AgentArtifacts{} agentManifests := &manifests.AgentManifests{} baseIso := &BaseIso{} - dependencies.Get(agentArtifacts, agentManifests, baseIso) + kubeconfig := &manifests.KubeConfigFile{} + dependencies.Get(agentArtifacts, agentManifests, baseIso, kubeconfig) a.cpuArch = agentArtifacts.CPUArch a.rendezvousIP = agentArtifacts.RendezvousIP a.tmpPath = agentArtifacts.TmpPath a.isoPath = agentArtifacts.ISOPath a.bootArtifactsBaseURL = agentArtifacts.BootArtifactsBaseURL + a.kubeconfig = kubeconfig volumeID, err := isoeditor.VolumeIdentifier(a.isoPath) if err != nil { @@ -64,7 +68,8 @@ func (a *AgentImage) Generate(dependencies asset.Parents) error { } a.volumeID = volumeID - a.platform = agentManifests.AgentClusterInstall.Spec.PlatformType + // a.platform = agentManifests.AgentClusterInstall.Spec.PlatformType + a.platform = "BareMetal" if a.platform == hiveext.ExternalPlatformType { // when the bootArtifactsBaseURL is specified, construct the custom rootfs URL if a.bootArtifactsBaseURL != "" { @@ -214,7 +219,11 @@ func (a *AgentImage) PersistToFile(directory string) error { return errors.New("cannot generate ISO image due to configuration errors") } - agentIsoFile := filepath.Join(directory, fmt.Sprintf(agentISOFilename, a.cpuArch)) + name := fmt.Sprintf(agentISOFilename, a.cpuArch) + if a.kubeconfig != nil { + name = "workers-" + name + } + agentIsoFile := filepath.Join(directory, name) // Remove symlink if it exists os.Remove(agentIsoFile) diff --git a/pkg/asset/agent/image/ignition.go b/pkg/asset/agent/image/ignition.go index f73dad66464..ed855e2f04a 100644 --- a/pkg/asset/agent/image/ignition.go +++ b/pkg/asset/agent/image/ignition.go @@ -88,6 +88,7 @@ func (a *Ignition) Dependencies() []asset.Asset { &agentconfig.AgentHosts{}, &mirror.RegistriesConf{}, &mirror.CaBundle{}, + &manifests.KubeConfigFile{}, } } @@ -97,7 +98,8 @@ func (a *Ignition) Generate(dependencies asset.Parents) error { agentConfigAsset := &agentconfig.AgentConfig{} agentHostsAsset := &agentconfig.AgentHosts{} extraManifests := &manifests.ExtraManifests{} - dependencies.Get(agentManifests, agentConfigAsset, agentHostsAsset, extraManifests) + kubeConfig := &manifests.KubeConfigFile{} + dependencies.Get(agentManifests, agentConfigAsset, agentHostsAsset, extraManifests, kubeConfig) pwd := &password.KubeadminPassword{} dependencies.Get(pwd) @@ -158,9 +160,15 @@ func (a *Ignition) Generate(dependencies asset.Parents) error { } a.CPUArch = *osImage.CPUArchitecture - clusterName := fmt.Sprintf("%s.%s", - agentManifests.ClusterDeployment.Spec.ClusterName, - agentManifests.ClusterDeployment.Spec.BaseDomain) + var clusterName string + + if kubeConfig.Config != nil { + clusterName = kubeConfig.Config.Clusters[0].Name + } else { + clusterName = fmt.Sprintf("%s.%s", + agentManifests.ClusterDeployment.Spec.ClusterName, + agentManifests.ClusterDeployment.Spec.BaseDomain) + } imageTypeISO := "full-iso" platform := agentManifests.AgentClusterInstall.Spec.PlatformType @@ -180,16 +188,23 @@ func (a *Ignition) Generate(dependencies asset.Parents) error { infraEnvID, osImage, infraEnv.Spec.Proxy, - imageTypeISO) + imageTypeISO, + agentConfigAsset.Config) err = bootstrap.AddStorageFiles(&config, "/", "agent/files", agentTemplateData) if err != nil { return err } + apiVipDnsname := "" + if kubeConfig.Config != nil { + url, _ := url.Parse(kubeConfig.Config.Clusters[0].Cluster.Server) + apiVipDnsname = url.Hostname() + } + rendezvousHostFile := ignition.FileFromString(rendezvousHostEnvPath, "root", 0644, - getRendezvousHostEnv(agentTemplateData.ServiceProtocol, nodeZeroIP)) + getRendezvousHostEnv(agentTemplateData.ServiceProtocol, nodeZeroIP, apiVipDnsname)) config.Storage.Files = append(config.Storage.Files, rendezvousHostFile) err = addBootstrapScripts(&config, agentManifests.ClusterImageSet.Spec.ReleaseImage) @@ -296,12 +311,25 @@ func getTemplateData(name, pullSecret, releaseImageList, releaseImage, infraEnvID string, osImage *models.OsImage, proxy *v1beta1.Proxy, - imageTypeISO string) *agentTemplateData { + imageTypeISO string, + agentConfig *agent.Config) *agentTemplateData { + + cpAgents := 0 + wrkAgents := 0 + + if agentClusterInstall != nil { + cpAgents = agentClusterInstall.Spec.ProvisionRequirements.ControlPlaneAgents + wrkAgents = agentClusterInstall.Spec.ProvisionRequirements.WorkerAgents + } else { + // Let's assume they are all workers + wrkAgents = len(agentConfig.Hosts) + } + return &agentTemplateData{ ServiceProtocol: "http", PullSecret: pullSecret, - ControlPlaneAgents: agentClusterInstall.Spec.ProvisionRequirements.ControlPlaneAgents, - WorkerAgents: agentClusterInstall.Spec.ProvisionRequirements.WorkerAgents, + ControlPlaneAgents: cpAgents, + WorkerAgents: wrkAgents, ReleaseImages: releaseImageList, ReleaseImage: releaseImage, ReleaseImageMirror: releaseImageMirror, @@ -315,7 +343,7 @@ func getTemplateData(name, pullSecret, releaseImageList, releaseImage, } } -func getRendezvousHostEnv(serviceProtocol, nodeZeroIP string) string { +func getRendezvousHostEnv(serviceProtocol, nodeZeroIP, apiVipDnsname string) string { serviceBaseURL := url.URL{ Scheme: serviceProtocol, Host: net.JoinHostPort(nodeZeroIP, "8090"), @@ -330,7 +358,8 @@ func getRendezvousHostEnv(serviceProtocol, nodeZeroIP string) string { return fmt.Sprintf(`NODE_ZERO_IP=%s SERVICE_BASE_URL=%s IMAGE_SERVICE_BASE_URL=%s -`, nodeZeroIP, serviceBaseURL.String(), imageServiceBaseURL.String()) +API_VIP_DNSNAME=%s +`, nodeZeroIP, serviceBaseURL.String(), imageServiceBaseURL.String(), apiVipDnsname) } func addStaticNetworkConfig(config *igntypes.Config, staticNetworkConfig []*models.HostStaticNetworkConfig) (err error) { diff --git a/pkg/asset/agent/image/ignition_test.go b/pkg/asset/agent/image/ignition_test.go index 1f20e995857..c9a6d1fb593 100644 --- a/pkg/asset/agent/image/ignition_test.go +++ b/pkg/asset/agent/image/ignition_test.go @@ -86,7 +86,7 @@ func TestIgnition_getTemplateData(t *testing.T) { } clusterName := "test-agent-cluster-install.test" - templateData := getTemplateData(clusterName, pullSecret, releaseImageList, releaseImage, releaseImageMirror, haveMirrorConfig, publicContainerRegistries, agentClusterInstall, infraEnvID, osImage, proxy, "minimal-iso") + templateData := getTemplateData(clusterName, pullSecret, releaseImageList, releaseImage, releaseImageMirror, haveMirrorConfig, publicContainerRegistries, agentClusterInstall, infraEnvID, osImage, proxy, "minimal-iso", nil) assert.Equal(t, clusterName, templateData.ClusterName) assert.Equal(t, "http", templateData.ServiceProtocol) assert.Equal(t, pullSecret, templateData.PullSecret) @@ -104,7 +104,7 @@ func TestIgnition_getTemplateData(t *testing.T) { func TestIgnition_getRendezvousHostEnv(t *testing.T) { nodeZeroIP := "2001:db8::dead:beef" - rendezvousHostEnv := getRendezvousHostEnv("http", nodeZeroIP) + rendezvousHostEnv := getRendezvousHostEnv("http", nodeZeroIP, "") assert.Equal(t, "NODE_ZERO_IP="+nodeZeroIP+"\nSERVICE_BASE_URL=http://["+nodeZeroIP+"]:8090/\nIMAGE_SERVICE_BASE_URL=http://["+nodeZeroIP+"]:8888/\n", rendezvousHostEnv) diff --git a/pkg/asset/agent/manifests/agentclusterinstall.go b/pkg/asset/agent/manifests/agentclusterinstall.go index 6112d5b9902..623d93de8ed 100644 --- a/pkg/asset/agent/manifests/agentclusterinstall.go +++ b/pkg/asset/agent/manifests/agentclusterinstall.go @@ -127,6 +127,7 @@ func (*AgentClusterInstall) Dependencies() []asset.Asset { return []asset.Asset{ &agent.OptionalInstallConfig{}, &agentconfig.AgentHosts{}, + &KubeConfigFile{}, } } @@ -134,7 +135,12 @@ func (*AgentClusterInstall) Dependencies() []asset.Asset { func (a *AgentClusterInstall) Generate(dependencies asset.Parents) error { installConfig := &agent.OptionalInstallConfig{} agentHosts := &agentconfig.AgentHosts{} - dependencies.Get(agentHosts, installConfig) + kubeconfig := &KubeConfigFile{} + dependencies.Get(agentHosts, installConfig, kubeconfig) + + if kubeconfig.Config != nil { + return nil + } if installConfig.Config != nil { var numberOfWorkers int = 0 diff --git a/pkg/asset/agent/manifests/agentpullsecret.go b/pkg/asset/agent/manifests/agentpullsecret.go index 3e3bd570e05..fc6c76b4618 100644 --- a/pkg/asset/agent/manifests/agentpullsecret.go +++ b/pkg/asset/agent/manifests/agentpullsecret.go @@ -42,15 +42,35 @@ func (*AgentPullSecret) Name() string { func (*AgentPullSecret) Dependencies() []asset.Asset { return []asset.Asset{ &agent.OptionalInstallConfig{}, + &KubeConfigFile{}, + &PlainPullSecret{}, } } // Generate generates the AgentPullSecret manifest. func (a *AgentPullSecret) Generate(dependencies asset.Parents) error { installConfig := &agent.OptionalInstallConfig{} - dependencies.Get(installConfig) + kubeconfig := &KubeConfigFile{} + plainps := &PlainPullSecret{} - if installConfig.Config != nil { + dependencies.Get(installConfig, kubeconfig, plainps) + + if kubeconfig.Config != nil && plainps.File != nil { + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: kubeconfig.Config.Clusters[0].Name, + Namespace: "cluster0", + }, + StringData: map[string]string{ + pullSecretKey: string(plainps.File.Data), + }, + } + a.Config = secret + } else if installConfig.Config != nil { secret := &corev1.Secret{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", diff --git a/pkg/asset/agent/manifests/clusterdeployment.go b/pkg/asset/agent/manifests/clusterdeployment.go index 2d7a1b62c23..ce207720055 100644 --- a/pkg/asset/agent/manifests/clusterdeployment.go +++ b/pkg/asset/agent/manifests/clusterdeployment.go @@ -37,13 +37,19 @@ func (*ClusterDeployment) Name() string { func (*ClusterDeployment) Dependencies() []asset.Asset { return []asset.Asset{ &agent.OptionalInstallConfig{}, + &KubeConfigFile{}, } } // Generate generates the ClusterDeployment manifest. func (cd *ClusterDeployment) Generate(dependencies asset.Parents) error { installConfig := &agent.OptionalInstallConfig{} - dependencies.Get(installConfig) + kubeconfig := &KubeConfigFile{} + dependencies.Get(installConfig, kubeconfig) + + if kubeconfig.Config != nil { + return nil + } if installConfig.Config != nil { clusterDeployment := &hivev1.ClusterDeployment{ diff --git a/pkg/asset/agent/manifests/infraenv.go b/pkg/asset/agent/manifests/infraenv.go index 2106c859ff2..8de978ee28b 100644 --- a/pkg/asset/agent/manifests/infraenv.go +++ b/pkg/asset/agent/manifests/infraenv.go @@ -42,6 +42,8 @@ func (*InfraEnv) Dependencies() []asset.Asset { return []asset.Asset{ &agent.OptionalInstallConfig{}, &agentconfig.AgentConfig{}, + &KubeConfigFile{}, + &SSHPublicKey{}, } } @@ -50,10 +52,39 @@ func (i *InfraEnv) Generate(dependencies asset.Parents) error { installConfig := &agent.OptionalInstallConfig{} agentConfig := &agentconfig.AgentConfig{} - dependencies.Get(installConfig, agentConfig) + kubeconfig := &KubeConfigFile{} + sshPublicKey := &SSHPublicKey{} + dependencies.Get(installConfig, kubeconfig, agentConfig, sshPublicKey) - if installConfig.Config != nil { - infraEnv := &aiv1beta1.InfraEnv{ + var infraEnv *aiv1beta1.InfraEnv + + if kubeconfig.Config != nil { + + cluster := kubeconfig.Config.Clusters[0] + + infraEnv = &aiv1beta1.InfraEnv{ + ObjectMeta: metav1.ObjectMeta{ + Name: cluster.Name, + Namespace: "cluster0", + }, + Spec: aiv1beta1.InfraEnvSpec{ + ClusterRef: &aiv1beta1.ClusterReference{ + Name: cluster.Name, + Namespace: "cluster0", + }, + SSHAuthorizedKey: strings.Trim(*sshPublicKey.Key, "|\n\t"), + PullSecretRef: &corev1.LocalObjectReference{ + Name: cluster.Name + "-pull-secret", + }, + NMStateConfigLabelSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "infraenvs.agent-install.openshift.io": cluster.Name, + }, + }, + }, + } + } else if installConfig.Config != nil { + infraEnv = &aiv1beta1.InfraEnv{ ObjectMeta: metav1.ObjectMeta{ Name: getInfraEnvName(installConfig), Namespace: getObjectMetaNamespace(installConfig), @@ -88,17 +119,18 @@ func (i *InfraEnv) Generate(dependencies asset.Parents) error { if agentConfig.Config != nil { infraEnv.Spec.AdditionalNTPSources = agentConfig.Config.AdditionalNTPSources } - i.Config = infraEnv + } - infraEnvData, err := yaml.Marshal(infraEnv) - if err != nil { - return errors.Wrap(err, "failed to marshal agent installer infraEnv") - } + i.Config = infraEnv - i.File = &asset.File{ - Filename: infraEnvFilename, - Data: infraEnvData, - } + infraEnvData, err := yaml.Marshal(infraEnv) + if err != nil { + return errors.Wrap(err, "failed to marshal agent installer infraEnv") + } + + i.File = &asset.File{ + Filename: infraEnvFilename, + Data: infraEnvData, } return i.finish() diff --git a/pkg/asset/agent/manifests/kubeconfigfile.go b/pkg/asset/agent/manifests/kubeconfigfile.go new file mode 100644 index 00000000000..1cfac4b4505 --- /dev/null +++ b/pkg/asset/agent/manifests/kubeconfigfile.go @@ -0,0 +1,61 @@ +package manifests + +import ( + "os" + + "github.com/openshift/installer/pkg/asset" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" + + clientcmd "k8s.io/client-go/tools/clientcmd/api/v1" +) + +type KubeConfigFile struct { + Config *clientcmd.Config + File *asset.File +} + +var _ asset.WritableAsset = (*KubeConfigFile)(nil) + +func (a *KubeConfigFile) Dependencies() []asset.Asset { + return []asset.Asset{} +} + +func (a *KubeConfigFile) Generate(dependencies asset.Parents) error { + return nil +} + +// Name returns the human-friendly name of the asset. +func (a *KubeConfigFile) Name() string { + return "Kubeconfig file" +} + +// Load returns the ISO from disk. +func (a *KubeConfigFile) Load(f asset.FileFetcher) (bool, error) { + + file, err := f.FetchByName("kubeconfig") + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + config := &clientcmd.Config{} + if err := yaml.Unmarshal(file.Data, &config); err != nil { + return false, errors.Wrapf(err, "failed to unmarshal kubeconfig") + } + + a.Config = config + a.File = file + + return true, nil +} + +// Files returns the files generated by the asset. +func (a *KubeConfigFile) Files() []*asset.File { + if a.File != nil { + return []*asset.File{a.File} + } + return []*asset.File{} +} diff --git a/pkg/asset/agent/manifests/nmstateconfig.go b/pkg/asset/agent/manifests/nmstateconfig.go index a48aee852b4..5a5e2637121 100644 --- a/pkg/asset/agent/manifests/nmstateconfig.go +++ b/pkg/asset/agent/manifests/nmstateconfig.go @@ -65,6 +65,7 @@ func (*NMStateConfig) Dependencies() []asset.Asset { return []asset.Asset{ &agentconfig.AgentHosts{}, &agent.OptionalInstallConfig{}, + &KubeConfigFile{}, } } @@ -73,13 +74,28 @@ func (n *NMStateConfig) Generate(dependencies asset.Parents) error { agentHosts := &agentconfig.AgentHosts{} installConfig := &agent.OptionalInstallConfig{} - dependencies.Get(agentHosts, installConfig) + kubeconfig := &KubeConfigFile{} + dependencies.Get(agentHosts, installConfig, kubeconfig) staticNetworkConfig := []*models.HostStaticNetworkConfig{} nmStateConfigs := []*aiv1beta1.NMStateConfig{} var data string var isNetworkConfigAvailable bool + mName := getNMStateConfigName(installConfig) + mNamespace := getObjectMetaNamespace(installConfig) + mLabels := getNMStateConfigLabels(installConfig) + + if kubeconfig.Config != nil { + cluster := kubeconfig.Config.Clusters[0] + + mName = cluster.Name + mNamespace = "cluster0" + mLabels = map[string]string{ + "infraenvs.agent-install.openshift.io": cluster.Name, + } + } + if len(agentHosts.Hosts) == 0 { return nil } @@ -97,9 +113,9 @@ func (n *NMStateConfig) Generate(dependencies asset.Parents) error { APIVersion: "agent-install.openshift.io/v1beta1", }, ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf(getNMStateConfigName(installConfig)+"-%d", i), - Namespace: getObjectMetaNamespace(installConfig), - Labels: getNMStateConfigLabels(installConfig), + Name: fmt.Sprintf(mName+"-%d", i), + Namespace: mNamespace, + Labels: mLabels, }, Spec: aiv1beta1.NMStateConfigSpec{ NetConfig: aiv1beta1.NetConfig{ @@ -376,6 +392,10 @@ func buildMacInterfaceMap(nmStateConfig aiv1beta1.NMStateConfig) models.MacInter } func validateHostCount(installConfig *types.InstallConfig, agentHosts *agentconfig.AgentHosts) error { + if installConfig == nil { + return nil + } + numRequiredMasters, numRequiredWorkers := agent.GetReplicaCount(installConfig) numMasters := int64(0) diff --git a/pkg/asset/agent/manifests/plainpullsecret.go b/pkg/asset/agent/manifests/plainpullsecret.go new file mode 100644 index 00000000000..6f3f47fda86 --- /dev/null +++ b/pkg/asset/agent/manifests/plainpullsecret.go @@ -0,0 +1,49 @@ +package manifests + +import ( + "os" + + "github.com/openshift/installer/pkg/asset" + "github.com/pkg/errors" +) + +type PlainPullSecret struct { + File *asset.File +} + +var _ asset.WritableAsset = (*PlainPullSecret)(nil) + +// Dependencies returns no dependencies. +func (a *PlainPullSecret) Dependencies() []asset.Asset { + return nil +} + +// Generate generates the SSH public key asset. +func (a *PlainPullSecret) Generate(asset.Parents) error { + return nil +} + +// Files returns the files generated by the asset. +func (a *PlainPullSecret) Files() []*asset.File { + if a.File != nil { + return []*asset.File{a.File} + } + return []*asset.File{} +} + +func (a *PlainPullSecret) Load(f asset.FileFetcher) (bool, error) { + file, err := f.FetchByName("pull-secret") + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, errors.Wrap(err, "failed to load pull-secret file") + } + a.File = file + return true, nil +} + +// Name returns the human-friendly name of the asset. +func (a PlainPullSecret) Name() string { + return "Plain pull secret" +} diff --git a/pkg/asset/agent/manifests/sshpublickey.go b/pkg/asset/agent/manifests/sshpublickey.go new file mode 100644 index 00000000000..c8619070889 --- /dev/null +++ b/pkg/asset/agent/manifests/sshpublickey.go @@ -0,0 +1,54 @@ +package manifests + +import ( + "os" + "path/filepath" + + "github.com/openshift/installer/pkg/asset" + "github.com/openshift/installer/pkg/validate" +) + +type SSHPublicKey struct { + Key *string +} + +var _ asset.Asset = (*SSHPublicKey)(nil) + +// Dependencies returns no dependencies. +func (a *SSHPublicKey) Dependencies() []asset.Asset { + return nil +} + +func readSSHKey(path string) (string, error) { + keyAsBytes, err := os.ReadFile(path) + if err != nil { + return "", err + } + + key := string(keyAsBytes) + + err = validate.SSHPublicKey(key) + if err != nil { + return "", err + } + + return key, nil +} + +// Generate generates the SSH public key asset. +func (a *SSHPublicKey) Generate(asset.Parents) error { + home := os.Getenv("HOME") + if home != "" { + key, err := readSSHKey(filepath.Join(home, ".ssh", "id_rsa.pub")) + if err == nil { + a.Key = &key + } + } + + return nil +} + +// Name returns the human-friendly name of the asset. +func (a SSHPublicKey) Name() string { + return "SSH Key" +} From 4b7d6117a5ce79fd0b0f184e2c72a9044c2ff42d Mon Sep 17 00:00:00 2001 From: Richard Su Date: Tue, 20 Feb 2024 21:02:57 -0500 Subject: [PATCH 2/3] [WIP] AGENT-850: day2 agent-import-cluster.service Add a new agent-import-cluster service for the addnodes workflow. --- .../usr/local/bin/get-container-images.sh | 2 + .../local/bin/start-cluster-installation.sh | 6 ++- .../assisted-service/images.env.template | 3 +- .../share/import-cluster/import-cluster.env | 3 ++ .../agent-import-cluster.service.template | 27 +++++++++++++ .../agent-register-infraenv.service.template | 2 +- pkg/asset/agent/image/ignition.go | 39 ++++++++++++++----- pkg/asset/agent/image/ignition_test.go | 2 +- pkg/asset/agent/image/kargs.go | 3 +- .../agent/image/unconfigured_ignition.go | 2 +- 10 files changed, 72 insertions(+), 17 deletions(-) create mode 100644 data/data/agent/files/usr/local/share/import-cluster/import-cluster.env create mode 100644 data/data/agent/systemd/units/agent-import-cluster.service.template diff --git a/data/data/agent/files/usr/local/bin/get-container-images.sh b/data/data/agent/files/usr/local/bin/get-container-images.sh index 13042a511bd..037d741b930 100644 --- a/data/data/agent/files/usr/local/bin/get-container-images.sh +++ b/data/data/agent/files/usr/local/bin/get-container-images.sh @@ -10,6 +10,8 @@ set -euo pipefail # The agent image will be also retrieved when its script is run cat </usr/local/share/assisted-service/agent-images.env SERVICE_IMAGE=$(image_for agent-installer-api-server) +# TODO: remove when assisted-service changes to import cluster merges +SERVICE_IMAGE=quay.io/rwsu1/assisted-service:latest CONTROLLER_IMAGE=$(image_for agent-installer-csr-approver) INSTALLER_IMAGE=$(image_for agent-installer-orchestrator) AGENT_DOCKER_IMAGE=$(image_for agent-installer-node-agent) diff --git a/data/data/agent/files/usr/local/bin/start-cluster-installation.sh b/data/data/agent/files/usr/local/bin/start-cluster-installation.sh index bd7e1f6228e..4c0a9120096 100644 --- a/data/data/agent/files/usr/local/bin/start-cluster-installation.sh +++ b/data/data/agent/files/usr/local/bin/start-cluster-installation.sh @@ -64,8 +64,10 @@ clear_issue "${status_issue}" sleep 1m # Install the host -echo "Installing host..." -curl -X POST ${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID}/actions/install +# echo "Installing host..." +# curl -X POST ${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID}/actions/install +# TODO: temporary to test cluster import without initiating install +echo "RWSU skip Installing host..." # while [[ "${cluster_status}" != "installed" ]] # do diff --git a/data/data/agent/files/usr/local/share/assisted-service/images.env.template b/data/data/agent/files/usr/local/share/assisted-service/images.env.template index 7df1f00b7f8..ba22a0cbfc3 100644 --- a/data/data/agent/files/usr/local/share/assisted-service/images.env.template +++ b/data/data/agent/files/usr/local/share/assisted-service/images.env.template @@ -1,3 +1,4 @@ ASSISTED_SERVICE_HOST=localhost:8090 ASSISTED_SERVICE_SCHEME={{.ServiceProtocol}} -OS_IMAGES=[{"openshift_version":"{{.OSImage.OpenshiftVersion}}","cpu_architecture":"{{.OSImage.CPUArchitecture}}","url":"{{.OSImage.URL}}","version":"{{.OSImage.Version}}"}] +# TODO: temporary to test on specific version +OS_IMAGES=[{"openshift_version":"{{.OSImage.OpenshiftVersion}}","cpu_architecture":"{{.OSImage.CPUArchitecture}}","url":"{{.OSImage.URL}}","version":"{{.OSImage.Version}}"},{"openshift_version":"4.15.0-rc.2","cpu_architecture":"x86_64","url":"https://rhcos.mirror.openshift.com/art/storage/prod/streams/4.14-9.2/builds/414.92.202307070025-0/x86_64/rhcos-414.92.202307070025-0-live.x86_64.iso","version":"414.92.202307070025-0"}] diff --git a/data/data/agent/files/usr/local/share/import-cluster/import-cluster.env b/data/data/agent/files/usr/local/share/import-cluster/import-cluster.env new file mode 100644 index 00000000000..f72c4d142a0 --- /dev/null +++ b/data/data/agent/files/usr/local/share/import-cluster/import-cluster.env @@ -0,0 +1,3 @@ +CLUSTER_ID=d8fc9f93-1b33-4b2a-943f-54afca271cb2 +CLUSTER_NAME=abi +CLUSTER_API_VIP_DNS_NAME=api.ostest.test.metalkube.org \ No newline at end of file diff --git a/data/data/agent/systemd/units/agent-import-cluster.service.template b/data/data/agent/systemd/units/agent-import-cluster.service.template new file mode 100644 index 00000000000..e27a4ff34eb --- /dev/null +++ b/data/data/agent/systemd/units/agent-import-cluster.service.template @@ -0,0 +1,27 @@ +[Unit] +Description=Service that imports the cluster +Wants=network-online.target assisted-service.service +PartOf=assisted-service-pod.service +After=network-online.target assisted-service.service +ConditionPathExists=/etc/assisted/node0 + +[Service] +Environment=PODMAN_SYSTEMD_UNIT=%n +Environment=OPENSHIFT_INSTALL_RELEASE_IMAGE_MIRROR={{.ReleaseImageMirror}} +EnvironmentFile=/etc/assisted/rendezvous-host.env +EnvironmentFile=/usr/local/share/assisted-service/agent-images.env +EnvironmentFile=/usr/local/share/import-cluster/import-cluster.env +ExecStartPre=/usr/local/bin/wait-for-assisted-service.sh +ExecStartPre=/bin/rm -f %t/%n.ctr-id +ExecStart=podman run --net host --cidfile=%t/%n.ctr-id --cgroups=no-conmon --log-driver=journald --rm --pod-id-file=%t/assisted-service-pod.pod-id --replace --name=agent-import-cluster -v /etc/assisted/manifests:/manifests -v /etc/assisted/extra-manifests:/extra-manifests -v /etc/pki/ca-trust:/etc/pki/ca-trust:z {{ if .HaveMirrorConfig }}-v /etc/containers:/etc/containers{{ end }} --env SERVICE_BASE_URL --env CLUSTER_ID --env CLUSTER_NAME --env CLUSTER_API_VIP_DNS_NAME $SERVICE_IMAGE /usr/local/bin/agent-installer-client importCluster +ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id +ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id + +KillMode=none +Type=oneshot +Restart=on-failure +RestartSec=30 +RemainAfterExit=true + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/data/data/agent/systemd/units/agent-register-infraenv.service.template b/data/data/agent/systemd/units/agent-register-infraenv.service.template index 63f16b9e67b..4722f0597f4 100644 --- a/data/data/agent/systemd/units/agent-register-infraenv.service.template +++ b/data/data/agent/systemd/units/agent-register-infraenv.service.template @@ -2,7 +2,7 @@ Description=Service that registers the infraenv Wants=network-online.target assisted-service.service PartOf=assisted-service-pod.service -After=network-online.target assisted-service.service agent-register-cluster.service +After=network-online.target assisted-service.service {{ if eq .WorkflowType "install" }}agent-register-cluster.service{{ end }}{{ if eq .WorkflowType "addnodes" }}agent-import-cluster.service{{ end }} ConditionPathExists=/etc/assisted/node0 [Service] diff --git a/pkg/asset/agent/image/ignition.go b/pkg/asset/agent/image/ignition.go index ed855e2f04a..19e26a62914 100644 --- a/pkg/asset/agent/image/ignition.go +++ b/pkg/asset/agent/image/ignition.go @@ -67,6 +67,7 @@ type agentTemplateData struct { Proxy *v1beta1.Proxy ConfigImageFiles string ImageTypeISO string + WorkflowType string } // Name returns the human-friendly name of the asset. @@ -124,11 +125,17 @@ func (a *Ignition) Generate(dependencies asset.Parents) error { }, } + agentWorkflowType := "addnodes" + nodeZeroIP, err := RetrieveRendezvousIP(agentConfigAsset.Config, agentHostsAsset.Hosts, agentManifests.NMStateConfigs) if err != nil { return err } + if agentWorkflowType == "addnodes" { + nodeZeroIP = "127.0.0.1" + } + logrus.Infof("The rendezvous host IP (node0 IP) is %s", nodeZeroIP) a.RendezvousIP = nodeZeroIP @@ -171,10 +178,10 @@ func (a *Ignition) Generate(dependencies asset.Parents) error { } imageTypeISO := "full-iso" - platform := agentManifests.AgentClusterInstall.Spec.PlatformType - if platform == hiveext.ExternalPlatformType { - imageTypeISO = "minimal-iso" - } + // platform := agentManifests.AgentClusterInstall.Spec.PlatformType + // if platform == hiveext.ExternalPlatformType { + // imageTypeISO = "minimal-iso" + // } agentTemplateData := getTemplateData( clusterName, @@ -189,7 +196,8 @@ func (a *Ignition) Generate(dependencies asset.Parents) error { osImage, infraEnv.Spec.Proxy, imageTypeISO, - agentConfigAsset.Config) + agentConfigAsset.Config, + agentWorkflowType) err = bootstrap.AddStorageFiles(&config, "/", "agent/files", agentTemplateData) if err != nil { @@ -233,7 +241,7 @@ func (a *Ignition) Generate(dependencies asset.Parents) error { return err } - enabledServices := getDefaultEnabledServices() + enabledServices := getDefaultEnabledServices(agentWorkflowType) // Enable pre-network-manager-config.service only when there are network configs defined if len(agentManifests.StaticNetworkConfigs) != 0 { enabledServices = append(enabledServices, "pre-network-manager-config.service") @@ -262,12 +270,11 @@ func (a *Ignition) Generate(dependencies asset.Parents) error { return nil } -func getDefaultEnabledServices() []string { - return []string{ +func getDefaultEnabledServices(workflowType string) []string { + enabledServices := []string{ "agent-interactive-console.service", "agent-interactive-console-serial@.service", "agent-register-infraenv.service", - "agent-register-cluster.service", "agent.service", "assisted-service-db.service", "assisted-service-pod.service", @@ -279,6 +286,16 @@ func getDefaultEnabledServices() []string { "set-hostname.service", "start-cluster-installation.service", } + + if workflowType == "install" { + enabledServices = append(enabledServices, "agent-register-cluster.service") + } + + if workflowType == "addnodes" { + enabledServices = append(enabledServices, "agent-import-cluster.service") + } + + return enabledServices } func addBootstrapScripts(config *igntypes.Config, releaseImage string) (err error) { @@ -312,7 +329,8 @@ func getTemplateData(name, pullSecret, releaseImageList, releaseImage, osImage *models.OsImage, proxy *v1beta1.Proxy, imageTypeISO string, - agentConfig *agent.Config) *agentTemplateData { + agentConfig *agent.Config, + workflowType string) *agentTemplateData { cpAgents := 0 wrkAgents := 0 @@ -340,6 +358,7 @@ func getTemplateData(name, pullSecret, releaseImageList, releaseImage, OSImage: osImage, Proxy: proxy, ImageTypeISO: imageTypeISO, + WorkflowType: workflowType, } } diff --git a/pkg/asset/agent/image/ignition_test.go b/pkg/asset/agent/image/ignition_test.go index c9a6d1fb593..648d6964e6e 100644 --- a/pkg/asset/agent/image/ignition_test.go +++ b/pkg/asset/agent/image/ignition_test.go @@ -86,7 +86,7 @@ func TestIgnition_getTemplateData(t *testing.T) { } clusterName := "test-agent-cluster-install.test" - templateData := getTemplateData(clusterName, pullSecret, releaseImageList, releaseImage, releaseImageMirror, haveMirrorConfig, publicContainerRegistries, agentClusterInstall, infraEnvID, osImage, proxy, "minimal-iso", nil) + templateData := getTemplateData(clusterName, pullSecret, releaseImageList, releaseImage, releaseImageMirror, haveMirrorConfig, publicContainerRegistries, agentClusterInstall, infraEnvID, osImage, proxy, "minimal-iso", nil, "install") assert.Equal(t, clusterName, templateData.ClusterName) assert.Equal(t, "http", templateData.ServiceProtocol) assert.Equal(t, pullSecret, templateData.PullSecret) diff --git a/pkg/asset/agent/image/kargs.go b/pkg/asset/agent/image/kargs.go index 1bd3decf9ac..15b57d0f917 100644 --- a/pkg/asset/agent/image/kargs.go +++ b/pkg/asset/agent/image/kargs.go @@ -33,7 +33,8 @@ func (a *Kargs) Generate(dependencies asset.Parents) error { a.consoleArgs = " console=ttyS0" } - a.fips = agentClusterInstall.FIPSEnabled() + // a.fips = agentClusterInstall.FIPSEnabled() + a.fips = false return nil } diff --git a/pkg/asset/agent/image/unconfigured_ignition.go b/pkg/asset/agent/image/unconfigured_ignition.go index 02b42da0f4a..319a35299e5 100644 --- a/pkg/asset/agent/image/unconfigured_ignition.go +++ b/pkg/asset/agent/image/unconfigured_ignition.go @@ -156,7 +156,7 @@ func (a *UnconfiguredIgnition) Generate(dependencies asset.Parents) error { return err } - enabledServices := getDefaultEnabledServices() + enabledServices := getDefaultEnabledServices("install") if len(nmStateConfigs.StaticNetworkConfig) > 0 { err = addStaticNetworkConfig(&config, nmStateConfigs.StaticNetworkConfig) if err != nil { From 2ad6b651ede46de766cab755b2472ee5b4389e39 Mon Sep 17 00:00:00 2001 From: Richard Su Date: Thu, 7 Mar 2024 11:05:22 -0500 Subject: [PATCH 3/3] Update apply-host-config service to pass WORKFLOW_TYPE --- .../agent/files/usr/local/bin/start-cluster-installation.sh | 2 -- .../usr/local/share/start-cluster/start-cluster.env.template | 2 +- ...-host-config.service => apply-host-config.service.template} | 3 ++- 3 files changed, 3 insertions(+), 4 deletions(-) rename data/data/agent/systemd/units/{apply-host-config.service => apply-host-config.service.template} (88%) diff --git a/data/data/agent/files/usr/local/bin/start-cluster-installation.sh b/data/data/agent/files/usr/local/bin/start-cluster-installation.sh index 4c0a9120096..c379647e263 100644 --- a/data/data/agent/files/usr/local/bin/start-cluster-installation.sh +++ b/data/data/agent/files/usr/local/bin/start-cluster-installation.sh @@ -61,8 +61,6 @@ done echo "All ${total_required_nodes} hosts are ready." 1>&2 clear_issue "${status_issue}" -sleep 1m - # Install the host # echo "Installing host..." # curl -X POST ${BASE_URL}/infra-envs/${INFRA_ENV_ID}/hosts/${HOST_ID}/actions/install diff --git a/data/data/agent/files/usr/local/share/start-cluster/start-cluster.env.template b/data/data/agent/files/usr/local/share/start-cluster/start-cluster.env.template index 285553d508a..af99c4c384b 100644 --- a/data/data/agent/files/usr/local/share/start-cluster/start-cluster.env.template +++ b/data/data/agent/files/usr/local/share/start-cluster/start-cluster.env.template @@ -1,2 +1,2 @@ REQUIRED_MASTER_NODES={{.ControlPlaneAgents}} -REQUIRED_WORKER_NODES={{.WorkerAgents}} +REQUIRED_WORKER_NODES=1 diff --git a/data/data/agent/systemd/units/apply-host-config.service b/data/data/agent/systemd/units/apply-host-config.service.template similarity index 88% rename from data/data/agent/systemd/units/apply-host-config.service rename to data/data/agent/systemd/units/apply-host-config.service.template index 5077ed9eab8..5a739a5752f 100644 --- a/data/data/agent/systemd/units/apply-host-config.service +++ b/data/data/agent/systemd/units/apply-host-config.service.template @@ -11,10 +11,11 @@ Environment=PODMAN_SYSTEMD_UNIT=%n EnvironmentFile=/etc/assisted/rendezvous-host.env EnvironmentFile=/usr/local/share/assisted-service/agent-images.env EnvironmentFile=/usr/local/share/assisted-service/assisted-service.env +Environment=WORKFLOW_TYPE={{.WorkflowType}} ExecStartPre=/bin/rm -f %t/%n.ctr-id ExecStartPre=/bin/mkdir -p %t/agent-installer /etc/assisted/hostconfig ExecStartPre=/usr/local/bin/wait-for-assisted-service.sh -ExecStart=podman run --net host --cidfile=%t/%n.ctr-id --cgroups=no-conmon --log-driver=journald --restart=on-failure:10 --pod-id-file=%t/assisted-service-pod.pod-id --replace --name=apply-host-config -v /etc/assisted/hostconfig:/etc/assisted/hostconfig -v %t/agent-installer:/var/run/agent-installer:z --env SERVICE_BASE_URL --env INFRA_ENV_ID $SERVICE_IMAGE /usr/local/bin/agent-installer-client configure +ExecStart=podman run --net host --cidfile=%t/%n.ctr-id --cgroups=no-conmon --log-driver=journald --restart=on-failure:10 --pod-id-file=%t/assisted-service-pod.pod-id --replace --name=apply-host-config -v /etc/assisted/hostconfig:/etc/assisted/hostconfig -v %t/agent-installer:/var/run/agent-installer:z --env SERVICE_BASE_URL --env INFRA_ENV_ID --env WORKFLOW_TYPE $SERVICE_IMAGE /usr/local/bin/agent-installer-client configure ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id