Skip to content

Commit

Permalink
manifest_install: Move manifest install to repo
Browse files Browse the repository at this point in the history
Move installation methods enum from helm repo
Move Manifest-related functionality from helm repo

Refs: #2105
Signed-off-by: Konstantin Yarovoy <[email protected]>
  • Loading branch information
Konstantin Yarovoy committed Jul 18, 2024
1 parent 7f8c910 commit 3e4001c
Show file tree
Hide file tree
Showing 8 changed files with 125 additions and 67 deletions.
2 changes: 1 addition & 1 deletion src/tasks/cleanup.cr
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ task "samples_cleanup" do |_, args|

config = CNFManager.parsed_config_file(cnf_config_file)
install_method = CNFManager.cnf_installation_method(config)
if install_method[0] == Helm::InstallMethod::ManifestDirectory
if install_method[0] == CNFInstall::InstallMethod::ManifestDirectory
installed_from_manifest = true
else
installed_from_manifest = false
Expand Down
2 changes: 1 addition & 1 deletion src/tasks/cnf_setup.cr
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ task "cnf_cleanup" do |_, args|
end
config = CNFManager.parsed_config_file(CNFManager.ensure_cnf_testsuite_yml_path(cnf))
install_method = CNFManager.cnf_installation_method(config)
if install_method[0] == Helm::InstallMethod::ManifestDirectory
if install_method[0] == CNFInstall::InstallMethod::ManifestDirectory
installed_from_manifest = true
else
installed_from_manifest = false
Expand Down
29 changes: 29 additions & 0 deletions src/tasks/utils/cnf_installation/common.cr
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
module CNFInstall
enum InstallMethod
HelmChart
HelmDirectory
ManifestDirectory
Invalid
end

def self.install_method_by_config_src(config_src : String) : InstallMethod
Log.info { "install_method_by_config_src" }
Log.info { "config_src: #{config_src}" }
helm_chart_file = "#{config_src}/#{Helm::CHART_YAML}"
Log.info { "looking for potential helm_chart_file: #{helm_chart_file}: file exists?: #{File.exists?(helm_chart_file)}" }

if !Dir.exists?(config_src)
Log.info { "install_method_by_config_src helm_chart selected" }
InstallMethod::HelmChart
elsif File.exists?(helm_chart_file)
Log.info { "install_method_by_config_src helm_directory selected" }
InstallMethod::HelmDirectory
elsif Dir.exists?(config_src)
Log.info { "install_method_by_config_src manifest_directory selected" }
InstallMethod::ManifestDirectory
else
puts "Error: #{config_src} is neither a helm_chart, helm_directory, or manifest_directory.".colorize(:red)
exit 1
end
end
end
51 changes: 51 additions & 0 deletions src/tasks/utils/cnf_installation/manifest.cr
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
module CNFInstall
module Manifest
def self.parse_manifest_as_ymls(template_file_name="cnfs/temp_template.yml")
Log.info { "parse_manifest_as_ymls template_file_name: #{template_file_name}" }
templates = File.read(template_file_name)
split_template = templates.split(/(\s|^)---(\s|$)/)
ymls = split_template.map { | template |
#TODO strip out NOTES
YAML.parse(template)
# compact seems to have problems with yaml::any
}.reject{|x|x==nil}
Log.debug { "read_template ymls: #{ymls}" }
ymls
end

def self.manifest_ymls_from_file_list(manifest_file_list)
ymls = manifest_file_list.map do |x|
parse_manifest_as_ymls(x)
end
ymls.flatten
end

def self.manifest_file_list(manifest_directory, silent=false)
Log.info { "manifest_file_list" }
Log.info { "manifest_directory: #{manifest_directory}" }
if manifest_directory && !manifest_directory.empty? && manifest_directory != "/"
cmd = "find #{manifest_directory}/ -name \"*.yml\" -o -name \"*.yaml\""
Log.info { cmd }
Process.run(
cmd,
shell: true,
output: find_resp = IO::Memory.new,
error: find_err = IO::Memory.new
)
manifests = find_resp.to_s.split("\n").select{|x| x.empty? == false}
Log.info { "find response: #{manifests}" }
if manifests.size == 0 && !silent
raise "No manifest ymls found in the #{manifest_directory} directory!"
end
manifests
else
[] of String
end
end

def self.manifest_containers(manifest_yml)
Log.debug { "manifest_containers: #{manifest_yml}" }
manifest_yml.dig?("spec", "template", "spec", "containers")
end
end
end
82 changes: 30 additions & 52 deletions src/tasks/utils/cnf_manager.cr
Original file line number Diff line number Diff line change
Expand Up @@ -89,20 +89,20 @@ module CNFManager
Log.debug { "install_method: #{install_method}" }
template_ymls = [] of YAML::Any
case install_method[0]
when Helm::InstallMethod::HelmChart, Helm::InstallMethod::HelmDirectory
when CNFInstall::InstallMethod::HelmChart, CNFInstall::InstallMethod::HelmDirectory
Log.info { "EXPORTED CHART PATH: #{helm_chart_path}" }
Helm.generate_manifest_from_templates(release_name,
helm_chart_path,
manifest_file_path,
helm_install_namespace,
helm_values)
template_ymls = Helm::Manifest.parse_manifest_as_ymls(manifest_file_path)
template_ymls = CNFInstall::Manifest.parse_manifest_as_ymls(manifest_file_path)
if !helm_install_namespace.empty?
default_namespace = config.cnf_config[:helm_install_namespace]
end
when Helm::InstallMethod::ManifestDirectory
when CNFInstall::InstallMethod::ManifestDirectory
# if release_name.empty? # no helm chart
template_ymls = Helm::Manifest.manifest_ymls_from_file_list(Helm::Manifest.manifest_file_list( destination_cnf_dir + "/" + manifest_directory))
template_ymls = CNFInstall::Manifest.manifest_ymls_from_file_list(CNFInstall::Manifest.manifest_file_list( destination_cnf_dir + "/" + manifest_directory))
# else
end

Expand Down Expand Up @@ -171,11 +171,11 @@ module CNFManager
helm_directory = config.cnf_config[:helm_directory]
manifest_directory = config.cnf_config[:manifest_directory]
case install_method[0]
when Helm::InstallMethod::ManifestDirectory
when CNFInstall::InstallMethod::ManifestDirectory
directory_parameters = directory_parameter_split(manifest_directory)["parameters"]
when Helm::InstallMethod::HelmChart
when CNFInstall::InstallMethod::HelmChart
directory_parameters = directory_parameter_split(helm_chart)["parameters"]
when Helm::InstallMethod::HelmDirectory
when CNFInstall::InstallMethod::HelmDirectory
directory_parameters = directory_parameter_split(helm_directory)["parameters"]
else
directory_parameters = ""
Expand Down Expand Up @@ -373,29 +373,7 @@ module CNFManager
end
end

def self.install_method_by_config_src(config_src : String, airgapped=false, generate_tar_mode=false)
Log.info { "install_method_by_config_src" }
Log.info { "config_src: #{config_src}" }
helm_chart_file = "#{config_src}/#{Helm::CHART_YAML}"
Log.info { "looking for potential helm_chart_file: #{helm_chart_file}: file exists?: #{File.exists?(helm_chart_file)}" }

if !Dir.exists?(config_src)
Log.info { "install_method_by_config_src helm_chart selected" }
Helm::InstallMethod::HelmChart
elsif File.exists?(helm_chart_file)
Log.info { "install_method_by_config_src helm_directory selected" }
Helm::InstallMethod::HelmDirectory
# elsif generate_tar_mode && KubectlClient::Apply.validate(config_src) # just because we are in generate tar mode doesn't mean we have a K8s cluster
elsif Dir.exists?(config_src)
Log.info { "install_method_by_config_src manifest_directory selected" }
Helm::InstallMethod::ManifestDirectory
else
puts "Error: #{config_src} is neither a helm_chart, helm_directory, or manifest_directory.".colorize(:red)
exit 1
end
end

def self.cnf_installation_method(config : CNFManager::Config) : Tuple(Helm::InstallMethod, String)
def self.cnf_installation_method(config : CNFManager::Config) : Tuple(CNFInstall::InstallMethod, String)
Log.info { "cnf_installation_method config : CNFManager::Config" }
Log.info { "config_cnf_config: #{config.cnf_config}" }
yml_file_path = config.cnf_config[:source_cnf_file]
Expand All @@ -421,7 +399,7 @@ module CNFManager
cnf_testsuite_helm_directory.split("/")[-1]
end
#Determine, for cnf, whether a helm chart, helm directory, or manifest directory is being used for installation
def self.cnf_installation_method(config : Totem::Config) : Tuple(Helm::InstallMethod, String)
def self.cnf_installation_method(config : Totem::Config) : Tuple(CNFInstall::InstallMethod, String)
Log.info { "cnf_installation_method" }
Log.info { "cnf_installation_method config: #{config}" }
Log.info { "cnf_installation_method config: #{config.config_paths[0]}/#{config.config_name}.#{config.config_type}" }
Expand Down Expand Up @@ -457,13 +435,13 @@ module CNFManager
end

if !helm_chart.empty?
{Helm::InstallMethod::HelmChart, helm_chart}
{CNFInstall::InstallMethod::HelmChart, helm_chart}
elsif !helm_directory.empty?
Log.info { "helm_directory not empty, using: #{full_helm_directory}" }
{Helm::InstallMethod::HelmDirectory, full_helm_directory}
{CNFInstall::InstallMethod::HelmDirectory, full_helm_directory}
elsif !manifest_directory.empty?
Log.info { "manifest_directory not empty, using: #{full_manifest_directory}" }
{Helm::InstallMethod::ManifestDirectory, full_manifest_directory}
{CNFInstall::InstallMethod::ManifestDirectory, full_manifest_directory}
else
puts "Error: Must populate at lease one installation type in #{config.config_paths[0]}/#{config.config_name}.#{config.config_type}: choose either helm_chart, helm_directory, or manifest_directory.".colorize(:red)
exit 1
Expand Down Expand Up @@ -525,11 +503,11 @@ module CNFManager
install_method = self.cnf_installation_method(config)
Log.debug { "install_method: #{install_method}" }
case install_method[0]
when Helm::InstallMethod::HelmChart
when CNFInstall::InstallMethod::HelmChart
Log.info { "generate_and_set_release_name install method: #{install_method[0]} data: #{install_method[1]}" }
Log.info { "generate_and_set_release_name helm_chart_or_directory: #{install_method[1]}" }
release_name = helm_chart_template_release_name(install_method[1], airgapped: airgapped)
when Helm::InstallMethod::HelmDirectory
when CNFInstall::InstallMethod::HelmDirectory
Log.info { "helm_directory install method: #{yml_path}/#{install_method[1]}" }
# todo if in airgapped mode, use path for airgapped repositories
# todo if in airgapped mode, get the release name
Expand All @@ -540,7 +518,7 @@ module CNFManager
else
release_name = helm_chart_template_release_name("#{install_method[1]}", airgapped: airgapped)
end
when Helm::InstallMethod::ManifestDirectory
when CNFInstall::InstallMethod::ManifestDirectory
Log.debug { "manifest_directory install method" }
release_name = UUID.random.to_s
else
Expand Down Expand Up @@ -668,7 +646,7 @@ module CNFManager
# todo manifest_or_helm_directory should either be the source helm/manifest files or the destination
# directory that they will be copied to/generated into, but *not both*
case install_method[0]
when Helm::InstallMethod::ManifestDirectory
when CNFInstall::InstallMethod::ManifestDirectory
Log.info { "preparing manifest_directory sandbox" }
source_directory = config_source_dir(config_file) + "/" + manifest_directory
src_path = Path[source_directory].expand.to_s
Expand All @@ -679,7 +657,7 @@ module CNFManager
rescue File::AlreadyExistsError
Log.info { "manifest sandbox dir already exists at #{destination_cnf_dir}/#{File.basename(src_path)}" }
end
when Helm::InstallMethod::HelmDirectory
when CNFInstall::InstallMethod::HelmDirectory
Log.info { "preparing helm_directory sandbox" }
source_directory = config_source_dir(config_file) + "/" + source_helm_directory.split(" ")[0] # todo support parameters separately
src_path = Path[source_directory].expand.to_s
Expand All @@ -693,7 +671,7 @@ module CNFManager
Log.info { "helm directory not found at #{src_path}" }
raise HelmDirectoryMissingError.new(src_path)
end
when Helm::InstallMethod::HelmChart
when CNFInstall::InstallMethod::HelmChart
Log.info { "preparing helm chart sandbox" }
FileUtils.mkdir_p(Path[destination_cnf_dir].expand.to_s + "/exported_chart")
end
Expand Down Expand Up @@ -875,7 +853,7 @@ module CNFManager
Log.for("sample_setup:install_method").info { "#{install_method[1]}" }
elapsed_time = Time.measure do
case install_method[0]
when Helm::InstallMethod::ManifestDirectory
when CNFInstall::InstallMethod::ManifestDirectory
# todo airgap_manifest_directory << prepare a manifest directory for deployment into an airgapped environment, put in airgap module
if input_file && !input_file.empty?
yaml_template_files = Find.find("#{destination_cnf_dir}/#{manifest_directory}",
Expand All @@ -887,15 +865,15 @@ module CNFManager
template_files.map{|x| AirGap.image_pull_policy(x)}
end
Log.for("verbose").info { "deploying by manifest file" } if verbose
file_list = Helm::Manifest.manifest_file_list(install_method[1], silent: false)
yml = Helm::Manifest.manifest_ymls_from_file_list(file_list)
file_list = CNFInstall::Manifest.manifest_file_list(install_method[1], silent: false)
yml = CNFInstall::Manifest.manifest_ymls_from_file_list(file_list)
if input_file && !input_file.empty?
image_pull(yml, "offline=true")
else
image_pull(yml, "offline=false")
end
KubectlClient::Apply.file("#{destination_cnf_dir}/#{manifest_directory}")
when Helm::InstallMethod::HelmChart
when CNFInstall::InstallMethod::HelmChart
if !helm_install_namespace.empty?
# default_namespace = config.cnf_config[:helm_install_namespace]
helm_install_namespace = config.cnf_config[:helm_install_namespace]
Expand All @@ -920,7 +898,7 @@ module CNFManager
Log.for("verbose").info { "deploying with chart repository" } if verbose
# Helm.template(release_name, install_method[1], output_file: "cnfs/temp_template.yml", values: helm_values)
Helm.template(release_name, install_method[1], output_file: "cnfs/temp_template.yml", namespace: helm_install_namespace, values: helm_values)
yml = Helm::Manifest.parse_manifest_as_ymls(template_file_name: "cnfs/temp_template.yml")
yml = CNFInstall::Manifest.parse_manifest_as_ymls(template_file_name: "cnfs/temp_template.yml")

if input_file && !input_file.empty?
image_pull(yml, "offline=true")
Expand All @@ -941,7 +919,7 @@ module CNFManager
fresh_install = false
end
export_published_chart(config, cli_args)
when Helm::InstallMethod::HelmDirectory
when CNFInstall::InstallMethod::HelmDirectory
if !helm_install_namespace.empty?
# default_namespace = config.cnf_config[:helm_install_namespace]
helm_install_namespace = config.cnf_config[:helm_install_namespace]
Expand All @@ -958,7 +936,7 @@ module CNFManager
#e.g. helm install nsm --set insecure=true ./nsm/helm_chart
# Helm.template(release_name, install_method[1], output_file: "cnfs/temp_template.yml")
Helm.template(release_name, install_method[1], output_file: "cnfs/temp_template.yml", namespace: helm_install_namespace, values: helm_values)
yml = Helm::Manifest.parse_manifest_as_ymls(template_file_name: "cnfs/temp_template.yml")
yml = CNFInstall::Manifest.parse_manifest_as_ymls(template_file_name: "cnfs/temp_template.yml")

if input_file && !input_file.empty?
image_pull(yml, "offline=true")
Expand Down Expand Up @@ -1141,9 +1119,9 @@ module CNFManager

Log.for("cnf_to_new_cluster").info { "Install method: #{install_method[0]}" }
case install_method[0]
when Helm::InstallMethod::ManifestDirectory
when CNFInstall::InstallMethod::ManifestDirectory
KubectlClient::Apply.file("#{destination_cnf_dir}/#{manifest_directory}", kubeconfig: kubeconfig)
when Helm::InstallMethod::HelmChart
when CNFInstall::InstallMethod::HelmChart
begin
if offline
chart_info = AirGap.tar_info_by_config_src(install_method[1])
Expand All @@ -1155,7 +1133,7 @@ module CNFManager
rescue e : Helm::CannotReuseReleaseNameError
stdout_warning "Release name #{release_name} has already been setup."
end
when Helm::InstallMethod::HelmDirectory
when CNFInstall::InstallMethod::HelmDirectory
begin
helm_install = Helm.install("#{release_name} #{destination_cnf_dir}/#{helm_directory} --kubeconfig #{kubeconfig} #{helm_namespace_option}")
rescue e : Helm::CannotReuseReleaseNameError
Expand Down Expand Up @@ -1224,7 +1202,7 @@ module CNFManager
install_method = self.cnf_installation_method(parsed_config)
Log.for("sample_cleanup:install_method").info { install_method }
case install_method[0]
when Helm::InstallMethod::HelmChart, Helm::InstallMethod::HelmDirectory
when CNFInstall::InstallMethod::HelmChart, CNFInstall::InstallMethod::HelmDirectory
helm_install_namespace = parsed_config.cnf_config[:helm_install_namespace]
if helm_install_namespace != nil && helm_install_namespace != ""
default_namespace = helm_install_namespace
Expand All @@ -1237,7 +1215,7 @@ module CNFManager
end
FileUtils.rm_rf(destination_cnf_dir)
return result[:status].success?
when Helm::InstallMethod::ManifestDirectory
when CNFInstall::InstallMethod::ManifestDirectory
manifest_directory = destination_cnf_dir + "/" + "#{config["manifest_directory"]? && config["manifest_directory"].as_s?}"
Log.for("cnf_cleanup:manifest_directory").info { manifest_directory }
result = KubectlClient::Delete.file("#{manifest_directory}", wait: true)
Expand Down
4 changes: 2 additions & 2 deletions src/tasks/utils/config.cr
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ module CNFManager
source_cnf_file: String,
source_cnf_dir: String,
yml_file_path: String,
install_method: Tuple(Helm::InstallMethod, String),
install_method: Tuple(CNFInstall::InstallMethod, String),

Check failure on line 21 in src/tasks/utils/config.cr

View workflow job for this annotation

GitHub Actions / Chaos & Oran Tests (pod_network_duplication)

undefined constant CNFInstall::InstallMethod

Check failure on line 21 in src/tasks/utils/config.cr

View workflow job for this annotation

GitHub Actions / Chaos & Oran Tests (pod_delete)

undefined constant CNFInstall::InstallMethod

Check failure on line 21 in src/tasks/utils/config.cr

View workflow job for this annotation

GitHub Actions / Chaos & Oran Tests (pod_memory_hog)

undefined constant CNFInstall::InstallMethod

Check failure on line 21 in src/tasks/utils/config.cr

View workflow job for this annotation

GitHub Actions / Chaos & Oran Tests (oran)

undefined constant CNFInstall::InstallMethod

Check failure on line 21 in src/tasks/utils/config.cr

View workflow job for this annotation

GitHub Actions / Chaos & Oran Tests (zombie)

undefined constant CNFInstall::InstallMethod

Check failure on line 21 in src/tasks/utils/config.cr

View workflow job for this annotation

GitHub Actions / Chaos & Oran Tests (disk_fill)

undefined constant CNFInstall::InstallMethod

Check failure on line 21 in src/tasks/utils/config.cr

View workflow job for this annotation

GitHub Actions / Chaos & Oran Tests (pod_network_latency)

undefined constant CNFInstall::InstallMethod

Check failure on line 21 in src/tasks/utils/config.cr

View workflow job for this annotation

GitHub Actions / Chaos & Oran Tests (pod_io_stress)

undefined constant CNFInstall::InstallMethod

Check failure on line 21 in src/tasks/utils/config.cr

View workflow job for this annotation

GitHub Actions / Chaos & Oran Tests (pod_network_corruption)

undefined constant CNFInstall::InstallMethod
manifest_directory: String,
helm_directory: String,
source_helm_directory: String,
Expand Down Expand Up @@ -220,7 +220,7 @@ module CNFManager
image_registry_fqdns: image_registry_fqdns,})

end
def self.install_method_by_config_file(config_file) : Helm::InstallMethod
def self.install_method_by_config_file(config_file) : CNFInstall::InstallMethod
LOGGING.info "install_data_by_config_file"
config = CNFManager.parsed_config_file(config_file)
sandbox_config = CNFManager::Config.parse_config_yml(CNFManager.ensure_cnf_testsuite_yml_path(config_file), airgapped: true, generate_tar_mode: false)
Expand Down
Loading

0 comments on commit 3e4001c

Please sign in to comment.